Пример #1
0
    def testEmulator(self):

        EmulatorHelper.setEmulators(True, True, True, True)
        self.assertEqual(PhEDEx().wrapped.__module__,
                         'WMQuality.Emulators.PhEDExClient.PhEDEx')
        self.assertEqual(DBSReader(self.globalDBS).wrapped.__module__,
                         'WMQuality.Emulators.DBSClient.DBSReader')
        self.assertEqual(SiteDBJSON().wrapped.__module__,
                         'WMQuality.Emulators.SiteDBClient.SiteDB')
        self.assertEqual(RequestManager().wrapped.__module__,
                         'WMQuality.Emulators.RequestManagerClient.RequestManager')

        self.assertEqual(PhEDEx().__class__.__name__, 'PhEDEx')
        self.assertEqual(DBSReader(self.globalDBS).__class__.__name__, 'DBSReader')
        self.assertEqual(SiteDBJSON().__class__.__name__, 'SiteDBJSON')
        self.assertEqual(RequestManager().__class__.__name__, 'RequestManager')

        EmulatorHelper.resetEmulators()
        self.assertEqual(PhEDEx().wrapped.__module__,
                         'WMCore.Services.PhEDEx.PhEDEx')
        self.assertEqual(DBSReader(self.globalDBS).wrapped.__module__,
                         'WMCore.Services.DBS.DBS2Reader')
        self.assertEqual(SiteDBJSON().wrapped.__module__,
                         'WMCore.Services.SiteDB.SiteDB')
        self.assertEqual(RequestManager().wrapped.__module__,
                         'WMCore.Services.RequestManager.RequestManager')

        self.assertEqual(PhEDEx().__class__.__name__, 'PhEDEx')
        self.assertEqual(DBSReader(self.globalDBS).__class__.__name__, 'DBS2Reader')
        self.assertEqual(SiteDBJSON().__class__.__name__, 'SiteDBJSON')
        self.assertEqual(RequestManager().__class__.__name__, 'RequestManager')
Пример #2
0
def getRequestInfoFromReqMgr(serviceURL):
    """ get the request info from requestManager """

    service = RequestManager({"endpoint": serviceURL})
    try:
        baseResults = service.getRequestNames()
        urls = service.getWorkQueue()
    except Exception, ex:
        logging.error(str(ex))
        return DFormatter.errorFormatter(serviceURL, "RequestManger Down")
def getSplitting(requestName):
    reqmgrUrl = 'https://cmsweb.cern.ch/reqmgr/reqMgr/'
    reqmgr = RequestManager(dict={'endpoint': reqmgrUrl})
    result = reqmgr.getRequest(requestName)
    workloadDB = Database(result['CouchWorkloadDBName'], result['CouchURL'])
    workloadPickle = workloadDB.getAttachment(requestName, 'spec')
    spec = pickle.loads(workloadPickle)
    workload = WMWorkloadHelper(spec)
    params = workload.getTopLevelTask()[0].jobSplittingParameters()
    algo = workload.getTopLevelTask()[0].jobSplittingAlgorithm()
    return params['events_per_job']
Пример #4
0
def main():
    demPolicy = Block()
    reqmgr = RequestManager(dict = {'endpoint' : 'https://cmsweb.cern.ch/reqmgr/reqMgr'})
    result = reqmgr.getRequest('pdmvserv_HIG-Summer12DR53X-01392_T1_ES_PIC_MSS_1_v0__130724_063344_7207')
    workloadDB = Database(result['CouchWorkloadDBName'], result['CouchURL'])
    workloadPickle = workloadDB.getAttachment('pdmvserv_HIG-Summer12DR53X-01392_T1_ES_PIC_MSS_1_v0__130724_063344_7207', 'spec')
    spec = pickle.loads(workloadPickle)
    workload = WMWorkloadHelper(spec)
    x,y = demPolicy(wmspec = workload, task = workload.getTopLevelTask()[0])
    print x
    print y
def getSplitting(requestName):
	reqmgrUrl='https://cmsweb.cern.ch/reqmgr/reqMgr/'
	reqmgr = RequestManager(dict = {'endpoint' : reqmgrUrl})
	result = reqmgr.getRequest(requestName)
	workloadDB = Database(result['CouchWorkloadDBName'], result['CouchURL'])
	workloadPickle = workloadDB.getAttachment(requestName, 'spec')
	spec = pickle.loads(workloadPickle)
	workload = WMWorkloadHelper(spec)
	params = workload.getTopLevelTask()[0].jobSplittingParameters()
	algo = workload.getTopLevelTask()[0].jobSplittingAlgorithm()
	return params['events_per_job']
Пример #6
0
    def __init__(self, config):
        """
        Initialise class members
        """
        BaseWorkerThread.__init__(self)

        myThread = threading.currentThread()
        self.daoFactory = DAOFactory(package="WMCore.WMBS",
                                     logger=myThread.logger,
                                     dbinterface=myThread.dbi)

        self.dbsDaoFactory = DAOFactory(package="WMComponent.DBS3Buffer",
                                        logger=myThread.logger,
                                        dbinterface=myThread.dbi)

        self.config = config
        self.jobCacheDir = self.config.JobCreator.jobCacheDir

        if getattr(self.config.TaskArchiver, "useWorkQueue", False) != False:
            # Get workqueue setup from config unless overridden
            if hasattr(self.config.TaskArchiver, 'WorkQueueParams'):
                self.workQueue = localQueue(
                    **self.config.TaskArchiver.WorkQueueParams)
            else:
                from WMCore.WorkQueue.WorkQueueUtils import queueFromConfig
                self.workQueue = queueFromConfig(self.config)
        else:
            self.workQueue = None

        self.timeout = getattr(self.config.TaskArchiver, "timeOut", None)
        self.useReqMgrForCompletionCheck = getattr(
            self.config.TaskArchiver, 'useReqMgrForCompletionCheck', True)

        if not self.useReqMgrForCompletionCheck:
            #sets the local monitor summary couch db
            self.requestLocalCouchDB = RequestDBWriter(
                self.config.AnalyticsDataCollector.localT0RequestDBURL,
                couchapp=self.config.AnalyticsDataCollector.RequestCouchApp)
            self.centralCouchDBWriter = self.requestLocalCouchDB
        else:
            self.centralCouchDBWriter = RequestDBWriter(
                self.config.AnalyticsDataCollector.centralRequestDBURL)

            self.reqmgr2Svc = ReqMgr(
                self.config.TaskArchiver.ReqMgr2ServiceURL)
            #TODO: remove this when reqmgr2 replace reqmgr completely (reqmgr2Only)
            self.reqmgrSvc = RequestManager(
                {'endpoint': self.config.TaskArchiver.ReqMgrServiceURL})

        #Load the cleanout state ID and save it
        stateIDDAO = self.daoFactory(classname="Jobs.GetStateID")
        self.stateID = stateIDDAO.execute("cleanout")

        return
Пример #7
0
def getSiteInfoFromReqMgr(serviceURL):
    """ get agent info from request mgr """

    reqMgr = RequestManager({'endpoint':serviceURL})
    #get information from global queue.
    try:
        queues = reqMgr.getWorkQueue()
    except Exception, ex:
        errorInfo = {}
        errorInfo['site_name'] = serviceURL
        return [errorInfo]
Пример #8
0
def getAgentInfoFromReqMgr(serviceURL):
    """ get agent info from request mgr """

    reqMgr = RequestManager({'endpoint':serviceURL})
    #get information from global queue.
    try:
        gQueues = reqMgr.getWorkQueue()
    except Exception, ex:
        errorInfo = {}
        errorInfo['url'] = serviceURL
        errorInfo['status'] = "Request Manager down: %s" % serviceURL
        errorInfo['acdc'] = 'N/A'
        return [errorInfo]
Пример #9
0
def getRequestInformationAndWorkload(requestName, reqmgrUrl):
    """
    _getRequestInformationAndWorkload_

    Retrieve the request information for assignment
    and the full pickled workload.
    """
    reqmgr = RequestManager(dict = {'endpoint' : reqmgrUrl})
    result = reqmgr.getRequest(requestName)
    workloadDB = Database(result['CouchWorkloadDBName'], result['CouchURL'])
    workloadPickle = workloadDB.getAttachment(requestName, 'spec')
    spec = pickle.loads(workloadPickle)
    workload = WMWorkloadHelper(spec)
    return workload, result
Пример #10
0
def getRequestInformationAndWorkload(requestName, reqmgrUrl):
    """
    _getRequestInformationAndWorkload_

    Retrieve the request information for assignment
    and the full pickled workload.
    """
    reqmgr = RequestManager(dict={'endpoint': reqmgrUrl})
    result = reqmgr.getRequest(requestName)
    workloadDB = Database(result['CouchWorkloadDBName'], result['CouchURL'])
    workloadPickle = workloadDB.getAttachment(requestName, 'spec')
    spec = pickle.loads(workloadPickle)
    workload = WMWorkloadHelper(spec)
    return workload, result
Пример #11
0
def getAgentInfoFromReqMgr(serviceURL):
    """ get agent info from request mgr """
    reqMgr = RequestManager({'endpoint': serviceURL})
    try:
        if serviceURL.lower() == "local":
            gQueues = getGlobalQueues()
        else:
            gQueues = reqMgr.getWorkQueue()
    except Exception, ex:
        logging.warning("Error: %s" % str(ex))
        errorInfo = {}
        errorInfo['url'] = serviceURL
        errorInfo['status'] = "Request Manager down: %s" % serviceURL
        errorInfo['acdc'] = 'N/A'
        return [errorInfo]
Пример #12
0
def getAgentInfoFromReqMgr(serviceURL):
    """ get agent info from request mgr """
    reqMgr = RequestManager({'endpoint':serviceURL})
    try:
        if serviceURL.lower() == "local":
            gQueues = getGlobalQueues()
        else:
            gQueues = reqMgr.getWorkQueue()
    except Exception, ex:
        logging.warning("Error: %s" % str(ex))
        errorInfo = {}
        errorInfo['url'] = serviceURL
        errorInfo['status'] = "Request Manager down: %s" % serviceURL
        errorInfo['acdc'] = 'N/A'
        return [errorInfo]
Пример #13
0
    def setup(self, parameters):
        """
        Called at startup
        """
        # set the connection for local couchDB call
        self.useReqMgrForCompletionCheck = getattr(
            self.config.TaskArchiver, 'useReqMgrForCompletionCheck', True)
        self.archiveDelayHours = getattr(self.config.TaskArchiver,
                                         'archiveDelayHours', 0)
        self.wmstatsCouchDB = WMStatsWriter(
            self.config.TaskArchiver.localWMStatsURL, "WMStatsAgent")

        #TODO: we might need to use local db for Tier0
        self.centralRequestDBReader = RequestDBReader(
            self.config.AnalyticsDataCollector.centralRequestDBURL,
            couchapp=self.config.AnalyticsDataCollector.RequestCouchApp)

        if self.useReqMgrForCompletionCheck:
            self.deletableState = "announced"
            self.centralRequestDBWriter = RequestDBWriter(
                self.config.AnalyticsDataCollector.centralRequestDBURL,
                couchapp=self.config.AnalyticsDataCollector.RequestCouchApp)
            if self.config.TaskArchiver.reqmgr2Only:
                self.reqmgr2Svc = ReqMgr(
                    self.config.TaskArchiver.ReqMgr2ServiceURL)
            else:
                #TODO: remove this for reqmgr2
                self.reqmgrSvc = RequestManager(
                    {'endpoint': self.config.TaskArchiver.ReqMgrServiceURL})
        else:
            # Tier0 case
            self.deletableState = "completed"
            # use local for update
            self.centralRequestDBWriter = RequestDBWriter(
                self.config.AnalyticsDataCollector.localT0RequestDBURL,
                couchapp=self.config.AnalyticsDataCollector.RequestCouchApp)

        jobDBurl = sanitizeURL(self.config.JobStateMachine.couchurl)['url']
        jobDBName = self.config.JobStateMachine.couchDBName
        self.jobCouchdb = CouchServer(jobDBurl)
        self.jobsdatabase = self.jobCouchdb.connectDatabase("%s/jobs" %
                                                            jobDBName)
        self.fwjrdatabase = self.jobCouchdb.connectDatabase("%s/fwjrs" %
                                                            jobDBName)

        statSummaryDBName = self.config.JobStateMachine.summaryStatsDBName
        self.statsumdatabase = self.jobCouchdb.connectDatabase(
            statSummaryDBName)
Пример #14
0
    def setup(self, parameters):
        """
        Called at startup
        """
        # set the connection for local couchDB call
        self.useReqMgrForCompletionCheck = getattr(
            self.config.TaskArchiver, 'useReqMgrForCompletionCheck', True)
        self.wmstatsCouchDB = WMStatsWriter(
            self.config.TaskArchiver.localWMStatsURL)
        self.centralCouchDBReader = WMStatsReader(
            self.config.TaskArchiver.centralWMStatsURL)

        if self.useReqMgrForCompletionCheck:
            self.deletableStates = ["announced"]
            self.centralCouchDBWriter = WMStatsWriter(
                self.config.TaskArchiver.centralWMStatsURL)
            self.reqmgrSvc = RequestManager(
                {'endpoint': self.config.TaskArchiver.ReqMgrServiceURL})
        else:
            # Tier0 case
            self.deletableStates = ["completed"]
            self.centralCouchDBWriter = self.wmstatsCouchDB

        jobDBurl = sanitizeURL(self.config.JobStateMachine.couchurl)['url']
        jobDBName = self.config.JobStateMachine.couchDBName
        self.jobCouchdb = CouchServer(jobDBurl)
        self.jobsdatabase = self.jobCouchdb.connectDatabase("%s/jobs" %
                                                            jobDBName)
        self.fwjrdatabase = self.jobCouchdb.connectDatabase("%s/fwjrs" %
                                                            jobDBName)

        statSummaryDBName = self.config.JobStateMachine.summaryStatsDBName
        self.statsumdatabase = self.jobCouchdb.connectDatabase(
            statSummaryDBName)
Пример #15
0
    def setUp(self):
        RESTBaseUnitTest.setUp(self)
        self.testInit.setupCouch("%s" % self.couchDBName, "GroupUser",
                                 "ConfigCache", "ReqMgr")
        self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats")
        # logging stuff from TestInit is broken, setting myself
        l = logging.getLogger()
        l.setLevel(logging.DEBUG)
        self.params = {}
        self.params['endpoint'] = self.config.getServerUrl()
        self.reqService = RequestManagerDS(self.params)
        self.jsonSender = JSONRequests(self.config.getServerUrl())

        userName = '******'
        groupName = 'Li'
        teamName = 'Tang'
        schema = utils.getAndSetupSchema(self,
                                         userName=userName,
                                         groupName=groupName,
                                         teamName=teamName)
        schema['ConfigCacheID'] = self.createConfig()
        schema['CouchDBName'] = self.couchDBName
        schema['CouchWorkloadDBName'] = self.couchDBName

        try:
            r = self.jsonSender.put('request', schema)
            try:
                self.requestName = r[0]['RequestName']
            except:
                self.requestName = r[0].values()[0]['RequestName']
        except Exception as ex:
            msg = traceback.format_exc()
            print "Exception during set up, reason: %s" % msg
            raise ex
Пример #16
0
    def setUp(self):
        RESTBaseUnitTest.setUp(self)
        self.testInit.setupCouch("%s" % self.couchDBName, "GroupUser", "ConfigCache", "ReqMgr")
        self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats")
        # logging stuff from TestInit is broken, setting myself
        l = logging.getLogger()
        l.setLevel(logging.DEBUG)
        self.params = {}
        self.params['endpoint'] = self.config.getServerUrl()
        self.reqService = RequestManagerDS(self.params)
        self.jsonSender = JSONRequests(self.config.getServerUrl())

        userName     = '******'
        groupName    = 'Li'
        teamName     = 'Tang'
        schema = utils.getAndSetupSchema(self,
                                         userName = userName,
                                         groupName = groupName,
                                         teamName = teamName)
        schema['ConfigCacheID'] = self.createConfig()
        schema['CouchDBName'] = self.couchDBName
        schema['CouchWorkloadDBName'] = self.couchDBName

        try:
            r = self.jsonSender.put('request', schema)
            try:
                self.requestName = r[0]['RequestName']
            except:
                self.requestName = r[0].values()[0]['RequestName']
        except Exception as ex:
            msg = traceback.format_exc()
            print("Exception during set up, reason: %s" % msg)
            raise ex
Пример #17
0
def getSiteInfoFromReqMgr(serviceURL):
    """ get agent info from request mgr """

    reqMgr = RequestManager({'endpoint':serviceURL})
    #get information from global queue.
    try:
        if serviceURL.lower() == "local":
            queues = getGlobalQueues()
        ###TODO: add back when GlobalMonitor spins out as a separate application
        else:
            queues = reqMgr.getWorkQueue()
    except Exception, ex:
        logging.warning("Error: %s" % str(ex))
        errorInfo = {}
        errorInfo['site_name'] = serviceURL
        return [errorInfo]
Пример #18
0
 def __init__(self, **kwargs):
     if not kwargs.get('logger'):
         import logging
         kwargs['logger'] = logging
     self.logger = kwargs['logger']
     self.reqMgr = RequestManager(kwargs)
     self.previous_state = {}
Пример #19
0
def getSiteInfoFromReqMgr(serviceURL):
    """ get agent info from request mgr """

    reqMgr = RequestManager({'endpoint': serviceURL})
    #get information from global queue.
    try:
        if serviceURL.lower() == "local":
            queues = getGlobalQueues()
        ###TODO: add back when GlobalMonitor spins out as a separate application
        else:
            queues = reqMgr.getWorkQueue()
    except Exception, ex:
        logging.warning("Error: %s" % str(ex))
        errorInfo = {}
        errorInfo['site_name'] = serviceURL
        return [errorInfo]
Пример #20
0
def getRequestInfoFromReqMgr(serviceURL):
    """ get the request info from requestManager """

    ###TODO: add back when GlobalMonitor spins out as a separate application
    service = RequestManager({'endpoint':serviceURL})
    try:
        ### use request manager funtion directly
        ### TODO: remove this when GlobalMonitor spins out as a separate application
        if serviceURL.lower() == "local":

            baseResults = getOverview()
            urls = getGlobalQueues()
        else:
            baseResults = service.getRequestNames()
            urls = service.getWorkQueue()
    except Exception, ex:
        logging.error(str(ex))
        return DFormatter.errorFormatter(serviceURL, "RequestManger Down")
Пример #21
0
    def setUp(self):
        """
        setUP global values
        Database setUp is done in base class
        """
        self.couchDBName = "reqmgr_t_0"
        RESTBaseUnitTest.setUp(self)
        self.testInit.setupCouch("%s" % self.couchDBName, "ConfigCache", "ReqMgr")
        self.testInit.setupCouch("%s_wmstats" % self.couchDBName,
                                 "WMStats")
        self.testInit.setupCouch("%s_acdc" % self.couchDBName,
                                 "ACDC", "GroupUser")
        reqMgrHost = self.config.getServerUrl()
        self.jsonSender = JSONRequests(reqMgrHost)

        self.params = {}
        self.params['endpoint'] = reqMgrHost
        self.reqService = RequestManager(self.params)
Пример #22
0
def getRequestInfoFromReqMgr(serviceURL):
    """ get the request info from requestManager """

    ###TODO: add back when GlobalMonitor spins out as a separate application
    service = RequestManager({'endpoint':serviceURL})
    try:
        ### use request manager funtion directly
        ### TODO: remove this when GlobalMonitor spins out as a separate application
        if serviceURL.lower() == "local":
            
            baseResults = getOverview()
            urls = getGlobalQueues()
        else:
            baseResults = service.getRequestNames()
            urls = service.getWorkQueue()
    except Exception, ex:
        logging.error(str(ex))
        return DFormatter.errorFormatter(serviceURL, "RequestManger Down")
Пример #23
0
    def __init__(self, **kwargs):
        if not kwargs.get('logger'):
            import logging
            kwargs['logger'] = logging
        self.logger = kwargs['logger']
        #TODO: (reqmgr2Only - remove this line when reqmgr is replaced)
        self.reqMgr = RequestManager(kwargs)
        #this will break all in one test
        self.reqMgr2 = ReqMgr(kwargs.get("reqmgr2_endpoint", None))

        centralurl = kwargs.get("central_logdb_url", "")
        identifier = kwargs.get("log_reporter", "")

        # set the thread name before creat the log db.
        # only sets that when it is not set already
        myThread = threading.currentThread()
        if myThread.getName() == "MainThread":
            myThread.setName(self.__class__.__name__)

        self.logdb = LogDB(centralurl, identifier, logger=self.logger)
        self.previous_state = {}
Пример #24
0
    def __init__(self, config):
        """
        Initialise class members
        """
        BaseWorkerThread.__init__(self)

        myThread = threading.currentThread()
        self.daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi)

        self.dbsDaoFactory = DAOFactory(
            package="WMComponent.DBS3Buffer", logger=myThread.logger, dbinterface=myThread.dbi
        )

        self.config = config
        self.jobCacheDir = self.config.JobCreator.jobCacheDir

        if getattr(self.config.TaskArchiver, "useWorkQueue", False) != False:
            # Get workqueue setup from config unless overridden
            if hasattr(self.config.TaskArchiver, "WorkQueueParams"):
                self.workQueue = localQueue(**self.config.TaskArchiver.WorkQueueParams)
            else:
                from WMCore.WorkQueue.WorkQueueUtils import queueFromConfig

                self.workQueue = queueFromConfig(self.config)
        else:
            self.workQueue = None

        self.timeout = getattr(self.config.TaskArchiver, "timeOut", None)
        self.useReqMgrForCompletionCheck = getattr(self.config.TaskArchiver, "useReqMgrForCompletionCheck", True)

        if not self.useReqMgrForCompletionCheck:
            # sets the local monitor summary couch db
            self.requestLocalCouchDB = RequestDBWriter(
                self.config.AnalyticsDataCollector.localT0RequestDBURL,
                couchapp=self.config.AnalyticsDataCollector.RequestCouchApp,
            )
            self.centralCouchDBWriter = self.requestLocalCouchDB
        else:
            self.centralCouchDBWriter = RequestDBWriter(self.config.AnalyticsDataCollector.centralRequestDBURL)

            self.reqmgr2Svc = ReqMgr(self.config.TaskArchiver.ReqMgr2ServiceURL)
            # TODO: remove this when reqmgr2 replace reqmgr completely (reqmgr2Only)
            self.reqmgrSvc = RequestManager({"endpoint": self.config.TaskArchiver.ReqMgrServiceURL})

        # Load the cleanout state ID and save it
        stateIDDAO = self.daoFactory(classname="Jobs.GetStateID")
        self.stateID = stateIDDAO.execute("cleanout")

        return
Пример #25
0
 def setUp(self):
     """
     setUP global values
     """
     RESTBaseUnitTest.setUp(self)
     self.params = {}
     self.params['endpoint'] = self.config.getServerUrl()
     self.reqService = RequestManagerDS(self.params)
     self.jsonSender = JSONRequests(self.config.getServerUrl())
     self.requestSchema = getRequestSchema()
     self.jsonSender.put('group/PeopleLikeMe')
     self.jsonSender.put('user/[email protected]')
     self.jsonSender.put('group/PeopleLikeMe/me')
     self.jsonSender.put('version/CMSSW_3_5_8')
     self.jsonSender.put('request/' + self.requestSchema['RequestName'], 
                         self.requestSchema)
Пример #26
0
    def setUp(self):
        """
        setUP global values
        Database setUp is done in base class
        """
        self.couchDBName = "reqmgr_t_0"
        RESTBaseUnitTest.setUp(self)
        self.testInit.setupCouch("%s" % self.couchDBName, "ConfigCache", "ReqMgr")
        self.testInit.setupCouch("%s_wmstats" % self.couchDBName,
                                 "WMStats")
        self.testInit.setupCouch("%s_acdc" % self.couchDBName,
                                 "ACDC", "GroupUser")
        reqMgrHost = self.config.getServerUrl()
        self.jsonSender = JSONRequests(reqMgrHost)

        self.params = {}
        self.params['endpoint'] = reqMgrHost
        self.reqService = RequestManager(self.params)
Пример #27
0
    def __init__(self, config):
        """
        __init__
        """
        BaseWorkerThread.__init__(self)
        self.config = config

        self.bossAir = BossAirAPI(config = self.config)
        self.reqmgr = RequestManager({'endpoint' : self.config.JobUpdater.reqMgrUrl})
        self.workqueue = WorkQueue(self.config.WorkQueueManager.couchurl,
                                   self.config.WorkQueueManager.dbname)

        myThread = threading.currentThread()

        self.daoFactory = DAOFactory(package = "WMCore.WMBS",
                                     logger = myThread.logger,
                                     dbinterface = myThread.dbi)

        self.listWorkflowsDAO = self.daoFactory(classname = "Workflow.ListForJobUpdater")
        self.updateWorkflowPrioDAO = self.daoFactory(classname = "Workflow.UpdatePriority")
        self.executingJobsDAO = self.daoFactory(classname = "Jobs.GetNumberOfJobsForWorkflowTaskStatus")
Пример #28
0
 def __init__(self, **kwargs):
     if not kwargs.get('logger'):
         import logging
         kwargs['logger'] = logging
     self.logger = kwargs['logger']
     self.reqMgr = RequestManager(kwargs)
     self.reqmgr2Only = kwargs.get("reqmgr2_only", False)
     #this will break all in one test
     self.reqMgr2 = ReqMgr(kwargs.get("reqmgr2_endpoint", None))
     
     centralurl = kwargs.get("central_logdb_url", "")
     identifier = kwargs.get("log_reporter", "")
     
     # set the thread name before creat the log db. 
     # only sets that when it is not set already
     myThread = threading.currentThread()
     if myThread.getName() == "MainThread":
         myThread.setName(self.__class__.__name__)
         
     self.logdb = LogDB(centralurl, identifier, logger=self.logger)
     self.previous_state = {}
Пример #29
0
    def __init__(self, config):
        """
        __init__
        """
        BaseWorkerThread.__init__(self)
        self.config = config

        self.bossAir = BossAirAPI(config = self.config)
        self.reqmgr = RequestManager({'endpoint' : self.config.JobUpdater.reqMgrUrl})
        self.workqueue = WorkQueue(self.config.WorkQueueManager.couchurl,
                                   self.config.WorkQueueManager.dbname)

        myThread = threading.currentThread()

        self.daoFactory = DAOFactory(package = "WMCore.WMBS",
                                     logger = myThread.logger,
                                     dbinterface = myThread.dbi)

        self.listWorkflowsDAO = self.daoFactory(classname = "Workflow.ListForJobUpdater")
        self.updateWorkflowPrioDAO = self.daoFactory(classname = "Workflow.UpdatePriority")
        self.executingJobsDAO = self.daoFactory(classname = "Jobs.GetNumberOfJobsForWorkflowTaskStatus")
Пример #30
0
 def setup(self, parameters):
     """
     Called at startup
     """
     self.teamName = self.config.Agent.teamName
     # set the connection for local couchDB call
     self.useReqMgrForCompletionCheck = getattr(self.config.TaskArchiver, 'useReqMgrForCompletionCheck', True)
     self.archiveDelayHours = getattr(self.config.TaskArchiver, 'archiveDelayHours', 0)
     self.wmstatsCouchDB = WMStatsWriter(self.config.TaskArchiver.localWMStatsURL, 
                                         "WMStatsAgent")
     
     #TODO: we might need to use local db for Tier0
     self.centralRequestDBReader = RequestDBReader(self.config.AnalyticsDataCollector.centralRequestDBURL, 
                                                   couchapp=self.config.AnalyticsDataCollector.RequestCouchApp)
     
     if self.useReqMgrForCompletionCheck:
         self.deletableState = "announced"
         self.centralRequestDBWriter = RequestDBWriter(self.config.AnalyticsDataCollector.centralRequestDBURL, 
                                                       couchapp=self.config.AnalyticsDataCollector.RequestCouchApp)
         if self.config.TaskArchiver.reqmgr2Only:
             self.reqmgr2Svc = ReqMgr(self.config.TaskArchiver.ReqMgr2ServiceURL)
         else:
             #TODO: remove this for reqmgr2
             self.reqmgrSvc = RequestManager({'endpoint': self.config.TaskArchiver.ReqMgrServiceURL})
     else:
         # Tier0 case
         self.deletableState = "completed"
         # use local for update
         self.centralRequestDBWriter = RequestDBWriter(self.config.AnalyticsDataCollector.localT0RequestDBURL, 
                                                       couchapp=self.config.AnalyticsDataCollector.RequestCouchApp)
     
     jobDBurl = sanitizeURL(self.config.JobStateMachine.couchurl)['url']
     jobDBName = self.config.JobStateMachine.couchDBName
     self.jobCouchdb  = CouchServer(jobDBurl)
     self.jobsdatabase = self.jobCouchdb.connectDatabase("%s/jobs" % jobDBName)
     self.fwjrdatabase = self.jobCouchdb.connectDatabase("%s/fwjrs" % jobDBName)
     
     statSummaryDBName = self.config.JobStateMachine.summaryStatsDBName
     self.statsumdatabase = self.jobCouchdb.connectDatabase(statSummaryDBName)
Пример #31
0
class RequestManagerTest(RESTBaseUnitTest):
    """
    Test RequestMgr Service client
    It will start RequestMgr RESTService
    Server DB is whatever env is set     
    
    This checks whether DS call makes without error and return the results.
    This test only test service call returns without error.
    The correctness of each function is tested in test/python/RequestManager_t/RequestMgr_t.py
    
    """
    def initialize(self):
        self.couchDBName = "reqmgr_t_0"
        self.config = RequestManagerConfig(
            'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel')
        dbUrl = os.environ.get("DATABASE", None)
        self.config.setDBUrl(dbUrl)
        self.config.setFormatter('WMCore.WebTools.RESTFormatter')
        self.config.setupRequestConfig()
        self.config.setupCouchDatabase(dbName=self.couchDBName)
        self.config.setPort(8899)
        self.schemaModules = ["WMCore.RequestManager.RequestDB"]

    def setUp(self):
        RESTBaseUnitTest.setUp(self)
        self.testInit.setupCouch("%s" % self.couchDBName, "GroupUser",
                                 "ConfigCache", "ReqMgr")
        self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats")
        # logging stuff from TestInit is broken, setting myself
        l = logging.getLogger()
        l.setLevel(logging.DEBUG)
        self.params = {}
        self.params['endpoint'] = self.config.getServerUrl()
        self.reqService = RequestManagerDS(self.params)
        self.jsonSender = JSONRequests(self.config.getServerUrl())

        userName = '******'
        groupName = 'Li'
        teamName = 'Tang'
        schema = utils.getAndSetupSchema(self,
                                         userName=userName,
                                         groupName=groupName,
                                         teamName=teamName)
        schema['ConfigCacheID'] = self.createConfig()
        schema['CouchDBName'] = self.couchDBName
        schema['CouchWorkloadDBName'] = self.couchDBName

        try:
            r = self.jsonSender.put('request', schema)
            try:
                self.requestName = r[0]['RequestName']
            except:
                self.requestName = r[0].values()[0]['RequestName']
        except Exception as ex:
            msg = traceback.format_exc()
            print "Exception during set up, reason: %s" % msg
            raise ex

    def tearDown(self):
        self.config.deleteWorkloadCache()
        RESTBaseUnitTest.tearDown(self)
        self.testInit.tearDownCouch()

    def createConfig(self, bad=False):
        """
        _createConfig_

        Create a config of some sort that we can load out of ConfigCache
        """
        PSetTweak = {
            'process': {
                'outputModules_': ['ThisIsAName'],
                'ThisIsAName': {
                    'dataset': {
                        'dataTier': 'RECO',
                        'filterName': 'Filter'
                    }
                }
            }
        }
        BadTweak = {
            'process': {
                'outputModules_': ['ThisIsAName1', 'ThisIsAName2'],
                'ThisIsAName1': {
                    'dataset': {
                        'dataTier': 'RECO',
                        'filterName': 'Filter'
                    }
                },
                'ThisIsAName2': {
                    'dataset': {
                        'dataTier': 'RECO',
                        'filterName': 'Filter'
                    }
                }
            }
        }
        configCache = ConfigCache(os.environ["COUCHURL"],
                                  couchDBName=self.couchDBName)
        configCache.createUserGroup(groupname="testGroup", username='******')
        if bad:
            configCache.setPSetTweaks(PSetTweak=BadTweak)
        else:
            configCache.setPSetTweaks(PSetTweak=PSetTweak)
        configCache.save()
        return configCache.getCouchID()

    @attr("integration")
    def testA_RequestManagerService(self):
        requestName = self.requestName

        request = self.reqService.getRequest(requestName)
        # minimal test : it's return type  and the some value inside
        self.assertEqual(type(request), dict)
        self.assertTrue(len(request) > 0)

        # Test putTeam
        self.reqService.putTeam("team_usa")
        self.assertTrue('team_usa' in self.jsonSender.get('team')[0])

        self.jsonSender.put('assignment/%s/%s' % ("team_usa", requestName))

        request = self.reqService.getAssignment(teamName="team_usa")
        self.assertEqual(type(request), list)
        self.assertTrue(len(request) > 0)

        request = self.reqService.getAssignment(request=requestName)
        self.assertEqual(type(request), list)
        self.assertTrue(len(request) > 0)

        self.reqService.sendMessage(requestName, "error")
        self.reqService.putWorkQueue(requestName, "http://test_url")
        self.reqService.reportRequestProgress(requestName,
                                              percent_complete=100,
                                              percent_success=90)

        self.reqService.updateRequestStatus(requestName, "running-open")
Пример #32
0
class ReqMgrTester(object):
    def __init__(self, reqMgrUrl):
        self.reqMgrUrl = reqMgrUrl
        self.restSender = JSONRequests(reqMgrUrl)
        d = dict(endpoint=self.reqMgrUrl)
        self.reqMgrService = RequestManager(d)

    def queryAllRequests(self):
        """
        Returns all requests stored at ReqMgr instance.
        
        """
        logging.info("Querying all requests at ReqMgr instance ...")
        r = self.reqMgrService.getRequestNames()
        print "Found %s requests:" % len(r)
        for req in r:
            print req

    def queryRequest(self, requestName):
        """
        Query a specific request according to the input argument.
        
        """
        urlQuery = "request/%s" % requestName
        logging.info("Querying request '%s'" % requestName)
        logging.info("Query: '%s':" % urlQuery)
        r = self.restSender.get(urlQuery)
        print str(r)

    def createRequests(self, numRequests):
        """
        Inject new numRequests into ReqMgr instance.
        (see ReqMgr_t testE how to create a request)
        
        """
        logging.info("Creating %s new requests ..." % numRequests)
        schema = ReReco.getTestArguments()
        schema['RequestName'] = 'TestReReco'
        schema['RequestType'] = 'ReReco'
        schema['CmsPath'] = "/uscmst1/prod/sw/cms"
        schema['Requestor'] = '%s' % "zmaxa"
        schema['Group'] = '%s' % "DATAOPS"
        schema['BlockWhitelist'] = ['/dataset/dataset/dataset#alpha']
        schema['BlockBlacklist'] = ['/dataset/dataset/dataset#beta']
        schema['Campaign'] = 'MyTestCampaign'
        for i in range(numRequests):
            urlQuery = "request/testRequest"
            print "Query: '%s':" % urlQuery
            print "Schema (request): '%s'" % schema
            r = self.restSender.put(urlQuery, schema)
            # print "request creating response: ", r
            print "created: ", r[0]["RequestName"]

    def deleteRequest(self, requestNames):
        """
        Delete requests specified in the input, more request names
        are comma-separated.
        
        """
        logging.info("Deleting requests ...")
        for reqName in requestNames.split(','):
            reqName = reqName.strip()
            urlQuery = "request/%s" % reqName
            logging.info("Deleting request (request_name): '%s'" % reqName)
            logging.info("Query: '%s':" % urlQuery)
            r = self.restSender.delete(urlQuery)

    def requestChangeStates(self, reqName):
        """
        Route the request (spec. by the request name) in the input
        through a series of possible request states.
        
        """
        logging.info("Changing state of request %s ..." % reqName)

        def changeState(requestName, urlQuery):
            logging.info("Query: '%s' ..." % urlQuery)
            r = self.restSender.put(urlQuery)
            r = self.restSender.get("request/%s" % requestName)
            #assert r[0]["RequestStatus"] == statusName
            logging.info("Querying modified request, new state: %s" %
                         r[0]["RequestStatus"])

        # once a request is created, it's in 'new' state
        # states transition has to be an allowed one as defined here:
        # WMCore/RequestManager/RequestDB/Settings/RequestStatus.py
        statesQueries = [
            "request/%s?status=%s" % (reqName, "testing-approved"),
            "request/%s?status=%s" % (reqName, "testing"),
            "request/%s?status=%s" % (reqName, "tested"),
            "request/%s?status=%s" % (reqName, "assignment-approved"),
            # create an assignment now
            # need quote because of space in the team name
            # (previous name - White Sox)
            urllib.quote("assignment/%s/%s" % (TEAM_NAME, reqName))
        ]
        for query in statesQueries:
            changeState(reqName, query)

    def setup(self):
        """
        Setup ReqMgr instance for dealing with requests - needs to create
        a user, group, SW releases entries, etc.
        as done in test/python/WMCore_t/RequestManager_t/ReqMgr_t.py
        
        """
        logging.info("ReqMgr setup ...")

        def doQuery(urlQuery):
            logging.info("Query: '%s' ..." % urlQuery)
            r = None
            try:
                r = self.restSender.put(urlQuery)
            except Exception as ex:
                print "exception"
                print str(ex)
            print "response:", r

        queries = [
            "user/[email protected]", "group/DATAOPS",
            "group/DATAOPS/zmaxa",
            urllib.quote("team/" + TEAM_NAME),
            "version/%s" % "CMSSW_3_5_8"
        ]
        for q in queries:
            doQuery(q)
        logging.info("ReqMgr setup finished, listing known users ...")
        q = "user/"
        r = self.restSender.get(q)
        print r
Пример #33
0
class TaskArchiverPoller(BaseWorkerThread):
    """
    Polls for Ended jobs

    List of attributes

    requireCouch:  raise an exception on couch failure instead of ignoring
    """
    def __init__(self, config):
        """
        Initialise class members
        """
        BaseWorkerThread.__init__(self)

        myThread = threading.currentThread()
        self.daoFactory = DAOFactory(package="WMCore.WMBS",
                                     logger=myThread.logger,
                                     dbinterface=myThread.dbi)

        self.dbsDaoFactory = DAOFactory(package="WMComponent.DBS3Buffer",
                                        logger=myThread.logger,
                                        dbinterface=myThread.dbi)

        self.config = config
        self.jobCacheDir = self.config.JobCreator.jobCacheDir

        if getattr(self.config.TaskArchiver, "useWorkQueue", False) != False:
            # Get workqueue setup from config unless overridden
            if hasattr(self.config.TaskArchiver, 'WorkQueueParams'):
                self.workQueue = localQueue(
                    **self.config.TaskArchiver.WorkQueueParams)
            else:
                from WMCore.WorkQueue.WorkQueueUtils import queueFromConfig
                self.workQueue = queueFromConfig(self.config)
        else:
            self.workQueue = None

        self.timeout = getattr(self.config.TaskArchiver, "timeOut", None)
        self.useReqMgrForCompletionCheck = getattr(
            self.config.TaskArchiver, 'useReqMgrForCompletionCheck', True)

        if not self.useReqMgrForCompletionCheck:
            #sets the local monitor summary couch db
            self.requestLocalCouchDB = RequestDBWriter(
                self.config.AnalyticsDataCollector.localT0RequestDBURL,
                couchapp=self.config.AnalyticsDataCollector.RequestCouchApp)
            self.centralCouchDBWriter = self.requestLocalCouchDB
        else:
            self.centralCouchDBWriter = RequestDBWriter(
                self.config.AnalyticsDataCollector.centralRequestDBURL)

            self.reqmgr2Svc = ReqMgr(
                self.config.TaskArchiver.ReqMgr2ServiceURL)
            #TODO: remove this when reqmgr2 replace reqmgr completely (reqmgr2Only)
            self.reqmgrSvc = RequestManager(
                {'endpoint': self.config.TaskArchiver.ReqMgrServiceURL})

        #Load the cleanout state ID and save it
        stateIDDAO = self.daoFactory(classname="Jobs.GetStateID")
        self.stateID = stateIDDAO.execute("cleanout")

        return

    def terminate(self, params):
        """
        _terminate_

        This function terminates the job after a final pass
        """
        logging.debug("terminating. doing one more pass before we die")
        self.algorithm(params)
        return

    def algorithm(self, parameters=None):
        """
        _algorithm_

        Executes the two main methods of the poller:
        1. findAndMarkFinishedSubscriptions
        2. completeTasks
        Final result is that finished workflows get their summary built and uploaded to couch,
        and all traces of them are removed from the agent WMBS and couch (this last one on demand).
        """
        try:
            self.findAndMarkFinishedSubscriptions()
            (finishedwfs, finishedwfsWithLogCollectAndCleanUp
             ) = self.getFinishedWorkflows()
            # set the data cache which can be used other thread (no ther thread should set the data cache)
            DataCache.setFinishedWorkflows(finishedwfsWithLogCollectAndCleanUp)
            self.completeTasks(finishedwfs)
        except WMException:
            myThread = threading.currentThread()
            if getattr(myThread, 'transaction', False) \
                   and getattr(myThread.transaction, 'transaction', False):
                myThread.transaction.rollback()
            raise
        except Exception as ex:
            myThread = threading.currentThread()
            msg = "Caught exception in TaskArchiver\n"
            msg += str(ex)
            if getattr(myThread, 'transaction', False) \
                   and getattr(myThread.transaction, 'transaction', False):
                myThread.transaction.rollback()
            raise TaskArchiverPollerException(msg)

        return

    def findAndMarkFinishedSubscriptions(self):
        """
        _findAndMarkFinishedSubscriptions_

        Find new finished subscriptions and mark as finished in WMBS.
        """
        myThread = threading.currentThread()

        myThread.transaction.begin()

        #Get the subscriptions that are now finished and mark them as such
        logging.info("Polling for finished subscriptions")
        finishedSubscriptions = self.daoFactory(
            classname="Subscriptions.MarkNewFinishedSubscriptions")
        finishedSubscriptions.execute(self.stateID, timeOut=self.timeout)
        logging.info("Finished subscriptions updated")

        myThread.transaction.commit()

        return

    def getFinishedWorkflows(self):
        """
        1. Get finished workflows (a finished workflow is defined in Workflow.GetFinishedWorkflows)
        2. Get finished workflows with logCollect and Cleanup only.
        3. combined those and make return 
           finishedwfs - without LogCollect and CleanUp task
           finishedwfsWithLogCollectAndCleanUp - including LogCollect and CleanUp task
        """

        finishedWorkflowsDAO = self.daoFactory(
            classname="Workflow.GetFinishedWorkflows")
        finishedwfs = finishedWorkflowsDAO.execute()
        finishedLogCollectAndCleanUpwfs = finishedWorkflowsDAO.execute(
            onlySecondary=True)
        finishedwfsWithLogCollectAndCleanUp = {}
        for wf in finishedLogCollectAndCleanUpwfs:
            if wf in finishedwfs:
                finishedwfsWithLogCollectAndCleanUp[wf] = finishedwfs[wf]
        return (finishedwfs, finishedwfsWithLogCollectAndCleanUp)

    def completeTasks(self, finishedwfs):
        """
        _completeTasks_

        This method will call several auxiliary methods to do the following:
        
        1. Notify the WorkQueue about finished subscriptions
        2. update dbsbuffer_workflow table with finished subscription
        """

        #Only delete those where the upload and notification succeeded
        logging.info("Found %d candidate workflows for completing: %s" %
                     (len(finishedwfs), finishedwfs.keys()))
        # update the completed flag in dbsbuffer_workflow table so blocks can be closed
        # create updateDBSBufferWorkflowComplete DAO
        if len(finishedwfs) == 0:
            return

        completedWorkflowsDAO = self.dbsDaoFactory(
            classname="UpdateWorkflowsToCompleted")

        centralCouchAlive = True
        try:
            #TODO: need to enable when reqmgr2 -wmstats is ready
            #abortedWorkflows = self.reqmgrCouchDBWriter.getRequestByStatus(["aborted"], format = "dict");
            abortedWorkflows = self.centralCouchDBWriter.getRequestByStatus(
                ["aborted"])
            logging.info(
                "There are %d requests in 'aborted' status in central couch." %
                len(abortedWorkflows))
            forceCompleteWorkflows = self.centralCouchDBWriter.getRequestByStatus(
                ["force-complete"])
            logging.info(
                "List of 'force-complete' workflows in central couch: %s" %
                forceCompleteWorkflows)

        except Exception as ex:
            centralCouchAlive = False
            logging.error(
                "we will try again when remote couch server comes back\n%s" %
                str(ex))

        if centralCouchAlive:
            for workflow in finishedwfs:
                try:
                    #Notify the WorkQueue, if there is one
                    if self.workQueue != None:
                        subList = []
                        logging.info("Marking subscriptions as Done ...")
                        for l in finishedwfs[workflow]["workflows"].values():
                            subList.extend(l)
                        self.notifyWorkQueue(subList)

                    #Now we know the workflow as a whole is gone, we can delete the information from couch
                    if not self.useReqMgrForCompletionCheck:
                        self.requestLocalCouchDB.updateRequestStatus(
                            workflow, "completed")
                        logging.info("status updated to completed %s" %
                                     workflow)

                    if workflow in abortedWorkflows:
                        #TODO: remove when reqmgr2-wmstats deployed
                        newState = "aborted-completed"
                    elif workflow in forceCompleteWorkflows:
                        newState = "completed"
                    else:
                        newState = None

                    if newState != None:
                        # update reqmgr workload document only request mgr is installed
                        if not self.useReqMgrForCompletionCheck:
                            # commented out untill all the agent is updated so every request have new state
                            # TODO: agent should be able to write reqmgr db diretly add the right group in
                            # reqmgr
                            self.requestLocalCouchDB.updateRequestStatus(
                                workflow, newState)
                        else:
                            try:
                                #TODO: try reqmgr1 call if it fails (reqmgr2Only - remove this line when reqmgr is replaced)
                                logging.info(
                                    "Updating status to '%s' in both oracle and couchdb ..."
                                    % newState)
                                self.reqmgrSvc.updateRequestStatus(
                                    workflow, newState)
                                #And replace with this - remove all the excption
                                #self.reqmgr2Svc.updateRequestStatus(workflow, newState)
                            except httplib.HTTPException as ex:
                                # If we get an HTTPException of 404 means reqmgr2 request
                                if ex.status == 404:
                                    # try reqmgr2 call
                                    msg = "%s : reqmgr2 request: %s" % (
                                        workflow, str(ex))
                                    logging.warning(msg)
                                    self.reqmgr2Svc.updateRequestStatus(
                                        workflow, newState)
                                else:
                                    msg = "%s : fail to update status %s  with HTTP error: %s" % (
                                        workflow, newState, str(ex))
                                    logging.error(msg)
                                    raise ex

                        logging.info("status updated to '%s' : %s" %
                                     (newState, workflow))

                    completedWorkflowsDAO.execute([workflow])

                except TaskArchiverPollerException as ex:

                    #Something didn't go well when notifying the workqueue, abort!!!
                    logging.error(
                        "Something bad happened while archiving tasks.")
                    logging.error(str(ex))
                    continue
                except Exception as ex:
                    #Something didn't go well on couch, abort!!!
                    msg = "Problem while archiving tasks for workflow %s\n" % workflow
                    msg += "Exception message: %s" % str(ex)
                    msg += "\nTraceback: %s" % traceback.format_exc()
                    logging.error(msg)
                    continue
        return

    def notifyWorkQueue(self, subList):
        """
        _notifyWorkQueue_

        Tells the workQueue component that a particular subscription,
        or set of subscriptions, is done.  Receives confirmation
        """

        for sub in subList:
            try:
                self.workQueue.doneWork(SubscriptionId=sub)
            except WorkQueueNoMatchingElements:
                #Subscription wasn't known to WorkQueue, feel free to clean up
                logging.debug(
                    "Local WorkQueue knows nothing about this subscription: %s"
                    % sub)
                pass
            except Exception as ex:
                msg = "Error talking to workqueue: %s\n" % str(ex)
                msg += "Tried to complete the following: %s\n" % sub
                raise TaskArchiverPollerException(msg)

        return
Пример #34
0
class CleanCouchPoller(BaseWorkerThread):
    """
    Cleans up local couch db according the the given condition.
    1. Cleans local couch db when request is completed and reported to cental db.
       This will clean up local couchdb, local summary db, local queue
       
    2. Cleans old couchdoc which is created older than the time threshold
    
    """
    def __init__(self, config):
        """
        Initialize config
        """
        BaseWorkerThread.__init__(self)
        # set the workqueue service for REST call
        self.config = config
        
    def setup(self, parameters):
        """
        Called at startup
        """
        self.teamName = self.config.Agent.teamName
        # set the connection for local couchDB call
        self.useReqMgrForCompletionCheck = getattr(self.config.TaskArchiver, 'useReqMgrForCompletionCheck', True)
        self.archiveDelayHours = getattr(self.config.TaskArchiver, 'archiveDelayHours', 0)
        self.wmstatsCouchDB = WMStatsWriter(self.config.TaskArchiver.localWMStatsURL, 
                                            "WMStatsAgent")
        
        #TODO: we might need to use local db for Tier0
        self.centralRequestDBReader = RequestDBReader(self.config.AnalyticsDataCollector.centralRequestDBURL, 
                                                      couchapp=self.config.AnalyticsDataCollector.RequestCouchApp)
        
        if self.useReqMgrForCompletionCheck:
            self.deletableState = "announced"
            self.centralRequestDBWriter = RequestDBWriter(self.config.AnalyticsDataCollector.centralRequestDBURL, 
                                                          couchapp=self.config.AnalyticsDataCollector.RequestCouchApp)
            if self.config.TaskArchiver.reqmgr2Only:
                self.reqmgr2Svc = ReqMgr(self.config.TaskArchiver.ReqMgr2ServiceURL)
            else:
                #TODO: remove this for reqmgr2
                self.reqmgrSvc = RequestManager({'endpoint': self.config.TaskArchiver.ReqMgrServiceURL})
        else:
            # Tier0 case
            self.deletableState = "completed"
            # use local for update
            self.centralRequestDBWriter = RequestDBWriter(self.config.AnalyticsDataCollector.localT0RequestDBURL, 
                                                          couchapp=self.config.AnalyticsDataCollector.RequestCouchApp)
        
        jobDBurl = sanitizeURL(self.config.JobStateMachine.couchurl)['url']
        jobDBName = self.config.JobStateMachine.couchDBName
        self.jobCouchdb  = CouchServer(jobDBurl)
        self.jobsdatabase = self.jobCouchdb.connectDatabase("%s/jobs" % jobDBName)
        self.fwjrdatabase = self.jobCouchdb.connectDatabase("%s/fwjrs" % jobDBName)
        
        statSummaryDBName = self.config.JobStateMachine.summaryStatsDBName
        self.statsumdatabase = self.jobCouchdb.connectDatabase(statSummaryDBName)

    def algorithm(self, parameters):
        """
        Get information from wmbs, workqueue and local couch.
          - It deletes old wmstats docs
          - Archive workflows
        """
        try:
            logging.info("Cleaning up the old request docs")
            report = self.wmstatsCouchDB.deleteOldDocs(self.config.TaskArchiver.DataKeepDays)
            logging.info("%s docs deleted" % report)

            # archiving only workflows that I own (same team)
            logging.info("Getting requests in '%s' state for team '%s'" % (self.deletableState,
                                                                           self.teamName))
            endTime = int(time.time()) - self.archiveDelayHours * 3600
            wfs = self.centralRequestDBReader.getRequestByTeamAndStatus(self.teamName,
                                                                        self.deletableState)
            commonWfs = self.centralRequestDBReader.getRequestByStatusAndStartTime(self.deletableState, 
                                                                                   False, endTime)
            deletableWorkflows = list(set(wfs) & set(commonWfs))
            logging.info("Ready to archive normal %s workflows" % len(deletableWorkflows))
            numUpdated = self.archiveWorkflows(deletableWorkflows, "normal-archived")
            logging.info("archive normal %s workflows" % numUpdated)
            
            abortedWorkflows = self.centralRequestDBReader.getRequestByStatus(["aborted-completed"])
            logging.info("Ready to archive aborted %s workflows" % len(abortedWorkflows))
            numUpdated = self.archiveWorkflows(abortedWorkflows, "aborted-archived")
            logging.info("archive aborted %s workflows" % numUpdated)
            
            rejectedWorkflows = self.centralRequestDBReader.getRequestByStatus(["rejected"])
            logging.info("Ready to archive rejected %s workflows" % len(rejectedWorkflows))
            numUpdated = self.archiveWorkflows(rejectedWorkflows, "rejected-archived")
            logging.info("archive rejected %s workflows" % numUpdated)

        except Exception as ex:
            logging.error(str(ex))
            logging.error("Error occurred, will try again next cycle")
    
    def archiveWorkflows(self, workflows, archiveState):
        updated = 0
        for workflowName in workflows:
            if self.cleanAllLocalCouchDB(workflowName):
                if self.useReqMgrForCompletionCheck:
                    
                    if self.config.TaskArchiver.reqmgr2Only:
                        self.reqmgr2Svc.updateRequestStatus(workflowName, archiveState)
                    else:
                        self.reqmgrSvc.updateRequestStatus(workflowName, archiveState);
                    updated += 1 
                    logging.debug("status updated to %s %s" % (archiveState, workflowName))
                else:
                    self.centralRequestDBWriter.updateRequestStatus(workflowName, archiveState)
        return updated
    
    def deleteWorkflowFromJobCouch(self, workflowName, db):
        """
        _deleteWorkflowFromCouch_

        If we are asked to delete the workflow from couch, delete it
        to clear up some space.

        Load the document IDs and revisions out of couch by workflowName,
        then order a delete on them.
        """
        if db == "JobDump":
            couchDB = self.jobsdatabase
            view = "jobsByWorkflowName"
        elif db == "FWJRDump":
            couchDB = self.fwjrdatabase
            view = "fwjrsByWorkflowName"
        elif db == "SummaryStats":
            couchDB = self.statsumdatabase
            view = None
        elif db == "WMStatsAgent":
            couchDB = self.wmstatsCouchDB.getDBInstance()
            view = "jobsByStatusWorkflow"
        
        if view == None:
            try:
                committed = couchDB.delete_doc(workflowName)
            except CouchNotFoundError as ex:
                return {'status': 'warning', 'message': "%s: %s" % (workflowName, str(ex))}
        else:
            options = {"startkey": [workflowName], "endkey": [workflowName, {}], "reduce": False}
            try:
                jobs = couchDB.loadView(db, view, options = options)['rows']
            except Exception as ex:
                errorMsg = "Error on loading jobs for %s" % workflowName
                logging.warning("%s/n%s" % (str(ex), errorMsg))
                return {'status': 'error', 'message': errorMsg}
            
            for j in jobs:
                doc = {}
                doc["_id"]  = j['value']['id']
                doc["_rev"] = j['value']['rev']
                couchDB.queueDelete(doc)
            committed = couchDB.commit()
        
        if committed:
            #create the error report
            errorReport = {}
            deleted = 0
            status = "ok"
            for data in committed:
                if 'error' in data:
                    errorReport.setdefault(data['error'], 0)
                    errorReport[data['error']] += 1
                    status = "error"
                else:
                    deleted += 1
            return {'status': status, 'delete': deleted, 'message': errorReport}
        else:
            return {'status': 'warning', 'message': "no %s exist" % workflowName}


    def cleanAllLocalCouchDB(self, workflowName):
        logging.info("Deleting %s from JobCouch" % workflowName)
        
        jobReport = self.deleteWorkflowFromJobCouch(workflowName, "JobDump")
        logging.debug("%s docs deleted from JobDump" % jobReport)
        
        fwjrReport = self.deleteWorkflowFromJobCouch(workflowName, "FWJRDump")
        logging.debug("%s docs deleted from FWJRDump" % fwjrReport)
        
        summaryReport = self.deleteWorkflowFromJobCouch(workflowName, "SummaryStats")
        logging.debug("%s docs deleted from SummaryStats" % summaryReport)
        
        wmstatsReport = self.deleteWorkflowFromJobCouch(workflowName, "WMStatsAgent")
        logging.debug("%s docs deleted from wmagent_summary" % wmstatsReport)
        
        # if one of the procedure fails return False
        if (jobReport["status"] == "error" or fwjrReport["status"] == "error" or 
            wmstatsReport["status"] == "error"):
            return False
        # other wise return True.
        return True
Пример #35
0
class WorkQueueReqMgrInterface():
    """Helper class for ReqMgr interaction"""
    def __init__(self, **kwargs):
        if not kwargs.get('logger'):
            import logging
            kwargs['logger'] = logging
        self.logger = kwargs['logger']
        #TODO: (reqmgr2Only - remove this line when reqmgr is replaced)
        self.reqMgr = RequestManager(kwargs)
        #this will break all in one test
        self.reqMgr2 = ReqMgr(kwargs.get("reqmgr2_endpoint", None))

        centralurl = kwargs.get("central_logdb_url", "")
        identifier = kwargs.get("log_reporter", "")

        # set the thread name before creat the log db.
        # only sets that when it is not set already
        myThread = threading.currentThread()
        if myThread.getName() == "MainThread":
            myThread.setName(self.__class__.__name__)

        self.logdb = LogDB(centralurl, identifier, logger=self.logger)
        self.previous_state = {}

    def __call__(self, queue):
        """Synchronize WorkQueue and RequestManager"""
        msg = ''
        try:  # pull in new work
            work = self.queueNewRequests(queue)
            msg += "New Work: %d\n" % work
        except Exception:
            self.logger.exception("Error caught during RequestManager pull")

        try:  # get additional open-running work
            extraWork = self.addNewElementsToOpenRequests(queue)
            msg += "Work added: %d\n" % extraWork
        except Exception:
            self.logger.exception("Error caught during RequestManager split")

        try:  # report back to ReqMgr
            uptodate_elements = self.report(queue)
            msg += "Updated ReqMgr status for: %s\n" % ", ".join(
                [x['RequestName'] for x in uptodate_elements])
        except Exception:
            self.logger.exception("Error caught during RequestManager update")
        else:
            try:  # Delete finished requests from WorkQueue
                self.deleteFinishedWork(queue, uptodate_elements)
            except Exception:
                self.logger.exception("Error caught during work deletion")

        queue.backend.recordTaskActivity('reqmgr_sync', msg)

    def queueNewRequests(self, queue):
        """Get requests from regMgr and queue to workqueue"""
        self.logger.info("Contacting Request manager for more work")
        work = 0
        workLoads = []

        if queue.params['DrainMode']:
            self.logger.info(
                'Draining queue: Skip requesting work from ReqMgr')
            return 0

        try:
            workLoads = self.getAvailableRequests(queue.params['Teams'])
        except Exception as ex:
            traceMsg = traceback.format_exc()
            msg = "Error contacting RequestManager: %s" % traceMsg
            self.logger.warning(msg)
            return 0

        for team, reqName, workLoadUrl in workLoads:
            #            try:
            #                self.reportRequestStatus(reqName, "negotiating")
            #            except Exception, ex:
            #                self.logger.error("""
            #                    Unable to update ReqMgr state to negotiating: %s
            #                    Ignoring this request: %s""" % (str(ex), reqName))
            #                continue

            try:
                try:
                    Lexicon.couchurl(workLoadUrl)
                except Exception as ex:  # can throw many errors e.g. AttributeError, AssertionError etc.
                    # check its not a local file
                    if not os.path.exists(workLoadUrl):
                        error = WorkQueueWMSpecError(
                            None,
                            "Workflow url validation error: %s" % str(ex))
                        raise error

                self.logger.info("Processing request %s at %s" %
                                 (reqName, workLoadUrl))
                units = queue.queueWork(workLoadUrl,
                                        request=reqName,
                                        team=team)
                self.logdb.delete(reqName, "error", this_thread=True)
            except (WorkQueueWMSpecError, WorkQueueNoWorkError) as ex:
                # fatal error - report back to ReqMgr
                self.logger.info(
                    'Permanent failure processing request "%s": %s' %
                    (reqName, str(ex)))
                self.logger.info("Marking request %s as failed in ReqMgr" %
                                 reqName)
                self.reportRequestStatus(reqName, 'Failed', message=str(ex))
                continue
            except (IOError, socket.error, CouchError,
                    CouchConnectionError) as ex:
                # temporary problem - try again later
                msg = 'Error processing request "%s": will try again later.' \
                '\nError: "%s"' % (reqName, str(ex))
                self.logger.info(msg)
                self.logdb.post(reqName, msg, 'error')
                continue
            except Exception as ex:
                # Log exception as it isnt a communication problem
                msg = 'Error processing request "%s": will try again later.' \
                '\nSee log for details.\nError: "%s"' % (reqName, str(ex))
                self.logger.exception('Unknown error processing %s' % reqName)
                self.logdb.post(reqName, msg, 'error')
                continue

            try:
                self.reportRequestStatus(reqName, "acquired")
            except Exception as ex:
                self.logger.warning("Unable to update ReqMgr state: %s" %
                                    str(ex))
                self.logger.warning('Will try again later')

            self.logger.info('%s units(s) queued for "%s"' % (units, reqName))
            work += units

            self.logger.info("%s element(s) obtained from RequestManager" %
                             work)
        return work

    def report(self, queue):
        """Report queue status to ReqMgr."""
        new_state = {}
        uptodate_elements = []
        now = time.time()

        elements = queue.statusInbox(dictKey="RequestName")
        if not elements:
            return new_state

        for ele in elements:
            ele = elements[ele][0]  # 1 element tuple
            try:
                request = self.reqMgr2.getRequestByNames(ele['RequestName'])
                if not request:
                    msg = 'Failed to get request "%s" from ReqMgr2. Will try again later.' % ele[
                        'RequestName']
                    self.logger.warning(msg)
                    continue
                request = request[ele['RequestName']]
                if request['RequestStatus'] in ('failed', 'completed',
                                                'announced', 'epic-FAILED',
                                                'closed-out', 'rejected'):
                    # requests can be done in reqmgr but running in workqueue
                    # if request has been closed but agent cleanup actions
                    # haven't been run (or agent has been retired)
                    # Prune out obviously too old ones to avoid build up
                    if queue.params.get('reqmgrCompleteGraceTime', -1) > 0:
                        if (now - float(ele.updatetime)
                            ) > queue.params['reqmgrCompleteGraceTime']:
                            # have to check all elements are at least running and are old enough
                            request_elements = queue.statusInbox(
                                WorkflowName=request['RequestName'])
                            if not any([
                                    x for x in request_elements
                                    if x['Status'] != 'Running'
                                    and not x.inEndState()
                            ]):
                                last_update = max([
                                    float(x.updatetime)
                                    for x in request_elements
                                ])
                                if (
                                        now - last_update
                                ) > queue.params['reqmgrCompleteGraceTime']:
                                    self.logger.info(
                                        "Finishing request %s as it is done in reqmgr"
                                        % request['RequestName'])
                                    queue.doneWork(
                                        WorkflowName=request['RequestName'])
                                    continue
                    else:
                        pass  # assume workqueue status will catch up later
                elif request['RequestStatus'] == 'aborted' or request[
                        'RequestStatus'] == 'force-complete':
                    queue.cancelWork(WorkflowName=request['RequestName'])
                # Check consistency of running-open/closed and the element closure status
                elif request['RequestStatus'] == 'running-open' and not ele.get(
                        'OpenForNewData', False):
                    self.reportRequestStatus(ele['RequestName'],
                                             'running-closed')
                elif request['RequestStatus'] == 'running-closed' and ele.get(
                        'OpenForNewData', False):
                    queue.closeWork(ele['RequestName'])
                # update request status if necessary
                elif ele['Status'] not in self._reqMgrToWorkQueueStatus(
                        request['RequestStatus']):
                    self.reportElement(ele)

                uptodate_elements.append(ele)
            except Exception as ex:
                msg = 'Error talking to ReqMgr about request "%s": %s'
                traceMsg = traceback.format_exc()
                self.logger.error(msg % (ele['RequestName'], traceMsg))

        return uptodate_elements

    def deleteFinishedWork(self, queue, elements):
        """Delete work from queue that is finished in ReqMgr"""
        finished = []
        for element in elements:
            if self._workQueueToReqMgrStatus(element['Status']) in ('aborted', 'failed', 'completed', 'announced',
                                                         'epic-FAILED', 'closed-out', 'rejected') \
                                                         and element.inEndState():
                finished.append(element['RequestName'])
        return queue.deleteWorkflows(*finished)

    def _getRequestsByTeamsAndStatus(self, status, teams=[]):
        """
        TODO: now it assumes one team per requests - check whether this assumption is correct
        Check whether we actually use the team for this.
        Also switch to byteamandstatus couch call instead of 
        """
        requests = self.reqMgr2.getRequestByStatus(status)
        #Then sort by Team name then sort by Priority
        #https://docs.python.org/2/howto/sorting.html
        if teams and len(teams) > 0:
            results = {}
            for reqName, value in requests.items():
                if value["Teams"][0] in teams:
                    results[reqName] = value
            return results
        else:
            return requests

    def getAvailableRequests(self, teams):
        """
        Get available requests for the given teams and sort by team and priority
        returns [(team, request_name, request_spec_url)]
        """

        tempResults = self._getRequestsByTeamsAndStatus("assigned",
                                                        teams).values()
        filteredResults = []
        for request in tempResults:
            if "Teams" in request and len(request["Teams"]) == 1:
                filteredResults.append(request)
                self.logdb.delete(request["RequestName"],
                                  "error",
                                  this_thread=True)
            else:
                msg = "no team or more than one team (%s) are assigined: %s" % (
                    request.get("Teams", None), request["RequestName"])
                self.logger.error(msg)
                self.logdb.post(request["RequestName"], msg, 'error')
        filteredResults.sort(key=itemgetter('RequestPriority'), reverse=True)
        filteredResults.sort(key=lambda r: r["Teams"][0])

        results = [(x["Teams"][0], x["RequestName"], x["RequestWorkflow"])
                   for x in filteredResults]

        return results

    def reportRequestStatus(self, request, status, message=None):
        """Change state in RequestManager
           Optionally, take a message to append to the request
        """
        if message:
            self.logdb.post(request, str(message), 'info')
        reqmgrStatus = self._workQueueToReqMgrStatus(status)
        if reqmgrStatus:  # only send known states
            try:
                #TODO: try reqmgr1 call if it fails (reqmgr2Only - remove this line when reqmgr is replaced)
                self.reqMgr.reportRequestStatus(request, reqmgrStatus)
                # And replace with this (remove all Exceptins)
                #self.reqMgr2.updateRequestStatus(request, reqmgrStatus)
            except HTTPException as ex:
                # If we get an HTTPException of 404 means reqmgr2 request
                if ex.status == 404:
                    # try reqmgr2 call
                    msg = "%s : reqmgr2 request: %s" % (request, str(ex))
                    self.logdb.post(request, msg, 'info')
                    self.reqMgr2.updateRequestStatus(request, reqmgrStatus)
                else:
                    msg = "%s : fail to update status with HTTP error: %s" % (
                        request, str(ex))
                    self.logdb.post(request, msg, 'warning')
                    raise ex
            except Exception as ex:
                msg = "%s : fail to update status will try later: %s" % (
                    request, str(ex))
                self.logdb.post(request, msg, 'warning')
                raise ex

    def markAcquired(self, request, url=None):
        """Mark request acquired"""
        self.reqMgr.putWorkQueue(request, url)

    def _workQueueToReqMgrStatus(self, status):
        """Map WorkQueue Status to that reported to ReqMgr"""
        statusMapping = {
            'Acquired': 'acquired',
            'Running': 'running-open',
            'Failed': 'failed',
            'Canceled': 'aborted',
            'CancelRequested': 'aborted',
            'Done': 'completed'
        }
        if status in statusMapping:
            # if wq status passed convert to reqmgr status
            return statusMapping[status]
        elif status in REQUEST_STATE_LIST:
            # if reqmgr status passed return reqmgr status
            return status
        else:
            # unknown status
            return None

    def _reqMgrToWorkQueueStatus(self, status):
        """Map ReqMgr status to that in a WorkQueue element, it is not a 1-1 relation"""
        statusMapping = {
            'acquired': ['Acquired'],
            'running': ['Running'],
            'running-open': ['Running'],
            'running-closed': ['Running'],
            'failed': ['Failed'],
            'aborted': ['Canceled', 'CancelRequested'],
            'force-complete': ['Canceled', 'CancelRequested'],
            'completed': ['Done']
        }
        if status in statusMapping:
            return statusMapping[status]
        else:
            return []

    def reportElement(self, element):
        """Report element to ReqMgr"""
        self.reportRequestStatus(element['RequestName'], element['Status'])

    def addNewElementsToOpenRequests(self, queue):
        """Add new elements to open requests which are in running-open state, only works adding new blocks from the input dataset"""
        self.logger.info(
            "Checking Request Manager for open requests and closing old ones")

        # First close any open inbox element which hasn't found anything new in a while
        queue.closeWork()
        self.report(queue)

        work = 0
        requests = []

        # Drain mode, don't pull any work into open requests. They will be closed if the queue stays in drain long enough
        if queue.params['DrainMode']:
            self.logger.info(
                'Draining queue: Skip requesting work from ReqMgr')
            return 0

        try:
            requests = self._getRequestsByTeamsAndStatus(
                "running-open", queue.params['Teams']).keys()
        except Exception as ex:
            traceMsg = traceback.format_exc()
            msg = "Error contacting RequestManager: %s" % traceMsg
            self.logger.warning(msg)
            return 0

        for reqName in requests:
            try:
                self.logger.info("Processing request %s" % (reqName))
                units = queue.addWork(requestName=reqName)
                self.logdb.delete(request["RequestName"], 'error', True)
            except (WorkQueueWMSpecError, WorkQueueNoWorkError) as ex:
                # fatal error - but at least it was split the first time. Log and skip.
                msg = 'Error adding further work to request "%s". Will try again later' \
                '\nError: "%s"' % (reqName, str(ex))
                self.logger.info(msg)
                self.logdb.post(reqName, msg, 'error')
                continue
            except (IOError, socket.error, CouchError,
                    CouchConnectionError) as ex:
                # temporary problem - try again later
                msg = 'Error processing request "%s": will try again later.' \
                '\nError: "%s"' % (reqName, str(ex))
                self.logger.info(msg)
                self.logdb.post(reqName, msg, 'error')
                continue
            except Exception as ex:
                # Log exception as it isnt a communication problem
                msg = 'Error processing request "%s": will try again later.' \
                '\nSee log for details.\nError: "%s"' % (reqName, str(ex))
                self.logger.exception('Unknown error processing %s' % reqName)
                self.logdb.post(reqName, msg, 'error')
                continue

            self.logger.info('%s units(s) queued for "%s"' % (units, reqName))
            work += units

        self.logger.info("%s element(s) added to open requests" % work)
        return work
Пример #36
0
class WorkQueueReqMgrInterface():
    """Helper class for ReqMgr interaction"""
    def __init__(self, **kwargs):
        if not kwargs.get('logger'):
            import logging
            kwargs['logger'] = logging
        self.logger = kwargs['logger']
        self.reqMgr = RequestManager(kwargs)
        self.reqmgr2Only = kwargs.get("reqmgr2_only", False)
        #this will break all in one test
        self.reqMgr2 = ReqMgr(kwargs.get("reqmgr2_endpoint", None))
        
        centralurl = kwargs.get("central_logdb_url", "")
        identifier = kwargs.get("log_reporter", "")
        
        # set the thread name before creat the log db. 
        # only sets that when it is not set already
        myThread = threading.currentThread()
        if myThread.getName() == "MainThread":
            myThread.setName(self.__class__.__name__)
            
        self.logdb = LogDB(centralurl, identifier, logger=self.logger)
        self.previous_state = {}

    def __call__(self, queue):
        """Synchronize WorkQueue and RequestManager"""
        msg = ''
        try:    # pull in new work
            work = self.queueNewRequests(queue)
            msg += "New Work: %d\n" % work
        except Exception:
            self.logger.exception("Error caught during RequestManager pull")

        try: # get additional open-running work
            extraWork = self.addNewElementsToOpenRequests(queue)
            msg += "Work added: %d\n" % extraWork
        except Exception:
            self.logger.exception("Error caught during RequestManager split")

        try:    # report back to ReqMgr
            uptodate_elements = self.report(queue)
            msg += "Updated ReqMgr status for: %s\n" % ", ".join([x['RequestName'] for x in uptodate_elements])
        except:
            self.logger.exception("Error caught during RequestManager update")
        else:
            try:    # Delete finished requests from WorkQueue
                self.deleteFinishedWork(queue, uptodate_elements)
            except:
                self.logger.exception("Error caught during work deletion")

        queue.backend.recordTaskActivity('reqmgr_sync', msg)

    def queueNewRequests(self, queue):
        """Get requests from regMgr and queue to workqueue"""
        self.logger.info("Contacting Request manager for more work")
        work = 0
        workLoads = []

        if queue.params['DrainMode']:
            self.logger.info('Draining queue: Skip requesting work from ReqMgr')
            return 0

        try:
            workLoads = self.getAvailableRequests(queue.params['Teams'])
        except Exception as ex:
            traceMsg = traceback.format_exc()
            msg = "Error contacting RequestManager: %s" % traceMsg
            self.logger.warning(msg)
            return 0

        for team, reqName, workLoadUrl in workLoads:
#            try:
#                self.reportRequestStatus(reqName, "negotiating")
#            except Exception, ex:
#                self.logger.error("""
#                    Unable to update ReqMgr state to negotiating: %s
#                    Ignoring this request: %s""" % (str(ex), reqName))
#                continue

            try:
                try:
                    Lexicon.couchurl(workLoadUrl)
                except Exception as ex: # can throw many errors e.g. AttributeError, AssertionError etc.
                    # check its not a local file
                    if not os.path.exists(workLoadUrl):
                        error = WorkQueueWMSpecError(None, "Workflow url validation error: %s" % str(ex))
                        raise error

                self.logger.info("Processing request %s at %s" % (reqName, workLoadUrl))
                units = queue.queueWork(workLoadUrl, request = reqName, team = team)
            except (WorkQueueWMSpecError, WorkQueueNoWorkError) as ex:
                # fatal error - report back to ReqMgr
                self.logger.info('Permanent failure processing request "%s": %s' % (reqName, str(ex)))
                self.logger.info("Marking request %s as failed in ReqMgr" % reqName)
                self.reportRequestStatus(reqName, 'Failed', message = str(ex))
                continue
            except (IOError, socket.error, CouchError, CouchConnectionError) as ex:
                # temporary problem - try again later
                msg = 'Error processing request "%s": will try again later.' \
                '\nError: "%s"' % (reqName, str(ex))
                self.logger.info(msg)
                self.logdb.post(reqName, msg, 'error')
                continue
            except Exception as ex:
                # Log exception as it isnt a communication problem
                msg = 'Error processing request "%s": will try again later.' \
                '\nSee log for details.\nError: "%s"' % (reqName, str(ex))
                self.logger.exception('Unknown error processing %s' % reqName)
                self.logdb.post(reqName, msg, 'error')
                continue

            try:
                self.reportRequestStatus(reqName, "acquired")
            except Exception as ex:
                self.logger.warning("Unable to update ReqMgr state: %s" % str(ex))
                self.logger.warning('Will try again later')

            self.logger.info('%s units(s) queued for "%s"' % (units, reqName))
            work += units

            self.logger.info("%s element(s) obtained from RequestManager" % work)
        return work


    def report(self, queue):
        """Report queue status to ReqMgr."""
        new_state = {}
        uptodate_elements = []
        now = time.time()

        elements = queue.statusInbox(dictKey = "RequestName")
        if not elements:
            return new_state

        for ele in elements:
            ele = elements[ele][0] # 1 element tuple
            try:
                request = self.reqMgr2.getRequestByNames(ele['RequestName'])[ele['RequestName']]
                if request['RequestStatus'] in ('failed', 'completed', 'announced',
                                                'epic-FAILED', 'closed-out', 'rejected'):
                    # requests can be done in reqmgr but running in workqueue
                    # if request has been closed but agent cleanup actions
                    # haven't been run (or agent has been retired)
                    # Prune out obviously too old ones to avoid build up
                    if queue.params.get('reqmgrCompleteGraceTime', -1) > 0:
                        if (now - float(ele.updatetime)) > queue.params['reqmgrCompleteGraceTime']:
                            # have to check all elements are at least running and are old enough
                            request_elements = queue.statusInbox(WorkflowName = request['RequestName'])
                            if not any([x for x in request_elements if x['Status'] != 'Running' and not x.inEndState()]):
                                last_update = max([float(x.updatetime) for x in request_elements])
                                if (now - last_update) > queue.params['reqmgrCompleteGraceTime']:
                                    self.logger.info("Finishing request %s as it is done in reqmgr" % request['RequestName'])
                                    queue.doneWork(WorkflowName=request['RequestName'])
                                    continue
                    else:
                        pass # assume workqueue status will catch up later
                elif request['RequestStatus'] == 'aborted' or request['RequestStatus'] == 'force-complete':
                    queue.cancelWork(WorkflowName=request['RequestName'])
                # Check consistency of running-open/closed and the element closure status
                elif request['RequestStatus'] == 'running-open' and not ele.get('OpenForNewData', False):
                    self.reportRequestStatus(ele['RequestName'], 'running-closed')
                elif request['RequestStatus'] == 'running-closed' and ele.get('OpenForNewData', False):
                    queue.closeWork(ele['RequestName'])
                # update request status if necessary
                elif ele['Status'] not in self._reqMgrToWorkQueueStatus(request['RequestStatus']):
                    self.reportElement(ele)
                    
                uptodate_elements.append(ele)
            except Exception as ex:
                msg = 'Error talking to ReqMgr about request "%s": %s'
                traceMsg = traceback.format_exc()
                self.logger.error(msg % (ele['RequestName'], traceMsg))

        return uptodate_elements

    def deleteFinishedWork(self, queue, elements):
        """Delete work from queue that is finished in ReqMgr"""
        finished = []
        for element in elements:
            if self._workQueueToReqMgrStatus(element['Status']) in ('aborted', 'failed', 'completed', 'announced',
                                                         'epic-FAILED', 'closed-out', 'rejected') \
                                                         and element.inEndState():
                finished.append(element['RequestName'])
        return queue.deleteWorkflows(*finished)
    
    def _getRequestsByTeamsAndStatus(self, status, teams = []):
        """
        TODO: now it assumes one team per requests - check whether this assumption is correct
        Check whether we actually use the team for this.
        Also switch to byteamandstatus couch call instead of 
        """
        requests = self.reqMgr2.getRequestByStatus(status)
        #Then sort by Team name then sort by Priority
        #https://docs.python.org/2/howto/sorting.html
        if teams and len(teams) > 0:
            results = {}
            for reqName, value in requests.items():
                if value["Teams"][0] in teams:
                    results[reqName] = value
            return results 
        else:
            return requests    
            
    def getAvailableRequests(self, teams):
        """
        Get available requests for the given teams and sort by team and priority
        returns [(team, request_name, request_spec_url)]
        """
        
        tempResults = self._getRequestsByTeamsAndStatus("assigned", teams).values()
        filteredResults = []
        for request in tempResults:
            if "Teams" in request and len(request["Teams"]) == 1:
                filteredResults.append(request)
            else:
                msg = "no team or more than one team (%s) are assigined: %s" % (
                                    request.get("Teams", None), request["RequestName"])
                self.logger.error(msg)
                self.logdb.post(request["RequestName"], msg, 'error')
        filteredResults.sort(key = itemgetter('RequestPriority'), reverse = True)
        filteredResults.sort(key = lambda r: r["Teams"][0])
        
        results = [(x["Teams"][0], x["RequestName"], x["RequestWorkflow"]) for x in filteredResults]
        
        return results

    def reportRequestStatus(self, request, status, message = None):
        """Change state in RequestManager
           Optionally, take a message to append to the request
        """
        if message:
            self.logdb.post(request, str(message), 'info')
        reqmgrStatus = self._workQueueToReqMgrStatus(status)
        if reqmgrStatus: # only send known states
            try:
                # try reqmgr1 call if it fails
                self.reqMgr.reportRequestStatus(request, reqmgrStatus)
            except Exception as ex:
                # try reqmgr2 call
                msg = "%s : reqmgr2 request: %s" % (request, str(ex))
                self.logdb.post(request, msg, 'warning')
                self.reqMgr2.updateRequestStatus(request, reqmgrStatus)                

    def markAcquired(self, request, url = None):
        """Mark request acquired"""
        self.reqMgr.putWorkQueue(request, url)

    def _workQueueToReqMgrStatus(self, status):
        """Map WorkQueue Status to that reported to ReqMgr"""
        statusMapping = {'Acquired' : 'acquired',
                         'Running' : 'running-open',
                         'Failed' : 'failed',
                         'Canceled' : 'aborted',
                         'CancelRequested' : 'aborted',
                         'Done' : 'completed'
                         }
        if status in statusMapping:
            # if wq status passed convert to reqmgr status
            return statusMapping[status]
        elif status in REQUEST_STATE_LIST:
            # if reqmgr status passed return reqmgr status
            return status
        else:
            # unknown status
            return None

    def _reqMgrToWorkQueueStatus(self, status):
        """Map ReqMgr status to that in a WorkQueue element, it is not a 1-1 relation"""
        statusMapping = {'acquired': ['Acquired'],
                         'running' : ['Running'],
                         'running-open': ['Running'],
                         'running-closed': ['Running'],
                         'failed': ['Failed'],
                         'aborted': ['Canceled', 'CancelRequested'],
                         'force-complete': ['Canceled', 'CancelRequested'],
                         'completed': ['Done']}
        if status in statusMapping:
            return statusMapping[status]
        else:
            return []

    def reportElement(self, element):
        """Report element to ReqMgr"""
        self.reportRequestStatus(element['RequestName'], element['Status'])

    def addNewElementsToOpenRequests(self, queue):
        """Add new elements to open requests which are in running-open state, only works adding new blocks from the input dataset"""
        self.logger.info("Checking Request Manager for open requests and closing old ones")

        # First close any open inbox element which hasn't found anything new in a while
        queue.closeWork()
        self.report(queue)

        work = 0
        requests = []

        # Drain mode, don't pull any work into open requests. They will be closed if the queue stays in drain long enough
        if queue.params['DrainMode']:
            self.logger.info('Draining queue: Skip requesting work from ReqMgr')
            return 0

        try:
            requests = self._getRequestsByTeamsAndStatus("running-open", queue.params['Teams']).keys()
        except Exception as ex:
            traceMsg = traceback.format_exc()
            msg = "Error contacting RequestManager: %s" % traceMsg
            self.logger.warning(msg)
            return 0

        for reqName in requests:
            try:
                self.logger.info("Processing request %s" % (reqName))
                units = queue.addWork(requestName = reqName)
            except (WorkQueueWMSpecError, WorkQueueNoWorkError) as ex:
                # fatal error - but at least it was split the first time. Log and skip.
                msg = 'Error adding further work to request "%s". Will try again later' \
                '\nError: "%s"' % (reqName, str(ex))
                self.logger.info(msg)
                self.logdb.post(reqName, msg, 'error')
                continue
            except (IOError, socket.error, CouchError, CouchConnectionError) as ex:
                # temporary problem - try again later
                msg = 'Error processing request "%s": will try again later.' \
                '\nError: "%s"' % (reqName, str(ex))
                self.logger.info(msg)
                self.logdb.post(reqName, msg, 'error')
                continue
            except Exception as ex:
                # Log exception as it isnt a communication problem
                msg = 'Error processing request "%s": will try again later.' \
                '\nSee log for details.\nError: "%s"' % (reqName, str(ex))
                self.logger.exception('Unknown error processing %s' % reqName)
                self.logdb.post(reqName, msg, 'error')
                continue

            self.logger.info('%s units(s) queued for "%s"' % (units, reqName))
            work += units

        self.logger.info("%s element(s) added to open requests" % work)
        return work
Пример #37
0
class CleanCouchPoller(BaseWorkerThread):
    """
    Cleans up local couch db according the the given condition.
    1. Cleans local couch db when request is completed and reported to cental db.
       This will clean up local couchdb, local summary db, local queue
       
    2. Cleans old couchdoc which is created older than the time threshold
    
    """
    def __init__(self, config):
        """
        Initialize config
        """
        BaseWorkerThread.__init__(self)
        # set the workqueue service for REST call
        self.config = config
        
    def setup(self, parameters):
        """
        Called at startup
        """
        # set the connection for local couchDB call
        self.useReqMgrForCompletionCheck   = getattr(self.config.TaskArchiver, 'useReqMgrForCompletionCheck', True)
        self.archiveDelayHours   = getattr(self.config.TaskArchiver, 'archiveDelayHours', 0)
        self.wmstatsCouchDB = WMStatsWriter(self.config.TaskArchiver.localWMStatsURL, 
                                            "WMStatsAgent")
        
        #TODO: we might need to use local db for Tier0
        self.centralRequestDBReader = RequestDBReader(self.config.AnalyticsDataCollector.centralRequestDBURL, 
                                                   couchapp = self.config.AnalyticsDataCollector.RequestCouchApp)
        
        if self.useReqMgrForCompletionCheck:
            self.deletableState = "announced"
            self.centralRequestDBWriter = RequestDBWriter(self.config.AnalyticsDataCollector.centralRequestDBURL, 
                                                   couchapp = self.config.AnalyticsDataCollector.RequestCouchApp)
            if self.config.TaskArchiver.reqmgr2Only:
                self.reqmgr2Svc = ReqMgr(self.config.TaskArchiver.ReqMgr2ServiceURL)
            else:
                #TODO: remove this for reqmgr2
                self.reqmgrSvc = RequestManager({'endpoint': self.config.TaskArchiver.ReqMgrServiceURL})
        else:
            # Tier0 case
            self.deletableState = "completed"
            # use local for update
            self.centralRequestDBWriter = RequestDBWriter(self.config.AnalyticsDataCollector.localT0RequestDBURL, 
                                                   couchapp = self.config.AnalyticsDataCollector.RequestCouchApp)
        
        jobDBurl = sanitizeURL(self.config.JobStateMachine.couchurl)['url']
        jobDBName = self.config.JobStateMachine.couchDBName
        self.jobCouchdb  = CouchServer(jobDBurl)
        self.jobsdatabase = self.jobCouchdb.connectDatabase("%s/jobs" % jobDBName)
        self.fwjrdatabase = self.jobCouchdb.connectDatabase("%s/fwjrs" % jobDBName)
        
        statSummaryDBName = self.config.JobStateMachine.summaryStatsDBName
        self.statsumdatabase = self.jobCouchdb.connectDatabase(statSummaryDBName)

    def algorithm(self, parameters):
        """
        get information from wmbs, workqueue and local couch
        """
        try:
            logging.info("Cleaning up the old request docs")
            report = self.wmstatsCouchDB.deleteOldDocs(self.config.TaskArchiver.DataKeepDays)
            logging.info("%s docs deleted" % report)
            logging.info("getting complete and announced requests")
            
            endTime = int(time.time()) - self.archiveDelayHours * 3600
            deletableWorkflows = self.centralRequestDBReader.getRequestByStatusAndStartTime(self.deletableState, 
                                                                                            False, endTime)
            logging.info("Ready to archive normal %s workflows" % len(deletableWorkflows))
            numUpdated = self.archiveWorkflows(deletableWorkflows, "normal-archived")
            logging.info("archive normal %s workflows" % numUpdated)
            
            abortedWorkflows = self.centralRequestDBReader.getRequestByStatus(["aborted-completed"])
            logging.info("Ready to archive aborted %s workflows" % len(abortedWorkflows))
            numUpdated = self.archiveWorkflows(abortedWorkflows, "aborted-archived")
            logging.info("archive aborted %s workflows" % numUpdated)
            
            rejectedWorkflows = self.centralRequestDBReader.getRequestByStatus(["rejected"])
            logging.info("Ready to archive rejected %s workflows" % len(rejectedWorkflows))
            numUpdated = self.archiveWorkflows(rejectedWorkflows, "rejected-archived")
            logging.info("archive rejected %s workflows" % numUpdated)

        except Exception as ex:
            logging.error(str(ex))
            logging.error("Error occurred, will try again next cycle")
    
    def archiveWorkflows(self, workflows, archiveState):
        updated = 0
        for workflowName in workflows:
            if self.cleanAllLocalCouchDB(workflowName):
                if self.useReqMgrForCompletionCheck:
                    
                    if self.config.TaskArchiver.reqmgr2Only:
                        self.reqmgr2Svc.updateRequestStatus(workflowName, archiveState)
                    else:
                        self.reqmgrSvc.updateRequestStatus(workflowName, archiveState);
                    updated += 1 
                    logging.debug("status updated to %s %s" % (archiveState, workflowName))
                else:
                    self.centralRequestDBWriter.updateRequestStatus(workflowName, archiveState)
        return updated
    
    def deleteWorkflowFromJobCouch(self, workflowName, db):
        """
        _deleteWorkflowFromCouch_

        If we are asked to delete the workflow from couch, delete it
        to clear up some space.

        Load the document IDs and revisions out of couch by workflowName,
        then order a delete on them.
        """
        if (db == "JobDump"):
            couchDB = self.jobsdatabase
            view = "jobsByWorkflowName"
        elif (db == "FWJRDump"):
            couchDB = self.fwjrdatabase
            view = "fwjrsByWorkflowName"
        elif (db == "SummaryStats"):
            couchDB = self.statsumdatabase
            view = None
        elif (db == "WMStats"):
            couchDB = self.wmstatsCouchDB.getDBInstance()
            view = "jobsByStatusWorkflow"
        
        if view == None:
            try:
                committed = couchDB.delete_doc(workflowName)
            except CouchNotFoundError as ex:
                return {'status': 'warning', 'message': "%s: %s" % (workflowName, str(ex))}
        else:
            options = {"startkey": [workflowName], "endkey": [workflowName, {}], "reduce": False}
            try:
                jobs = couchDB.loadView(db, view, options = options)['rows']
            except Exception as ex:
                errorMsg = "Error on loading jobs for %s" % workflowName
                logging.warning("%s/n%s" % (str(ex), errorMsg))
                return {'status': 'error', 'message': errorMsg}
            
            for j in jobs:
                doc = {}
                doc["_id"]  = j['value']['id']
                doc["_rev"] = j['value']['rev']
                couchDB.queueDelete(doc)
            committed = couchDB.commit()
        
        if committed:
            #create the error report
            errorReport = {}
            deleted = 0
            status = "ok"
            for data in committed:
                if 'error' in data:
                    errorReport.setdefault(data['error'], 0)
                    errorReport[data['error']] += 1
                    status = "error"
                else:
                    deleted += 1
            return {'status': status, 'delete': deleted, 'message': errorReport}
        else:
            return {'status': 'warning', 'message': "no %s exist" % workflowName}


    def cleanAllLocalCouchDB(self, workflowName):
        logging.info("Deleting %s from JobCouch" % workflowName)
        
        jobReport = self.deleteWorkflowFromJobCouch(workflowName, "JobDump")
        logging.debug("%s docs deleted from JobDump" % jobReport)
        
        fwjrReport = self.deleteWorkflowFromJobCouch(workflowName, "FWJRDump")
        logging.debug("%s docs deleted from FWJRDump" % fwjrReport)
        
        summaryReport = self.deleteWorkflowFromJobCouch(workflowName, "SummaryStats")
        logging.debug("%s docs deleted from SummaryStats" % summaryReport)
        
        wmstatsReport = self.deleteWorkflowFromJobCouch(workflowName, "WMStats")
        logging.debug("%s docs deleted from wmagent_summary" % wmstatsReport)
        
        # if one of the procedure fails return False
        if (jobReport["status"] == "error" or fwjrReport["status"] == "error" or 
            wmstatsReport["status"] == "error"):
            return False
        # other wise return True.
        return True
        
Пример #38
0
class ReqMgrTester(object):
    def __init__(self, reqMgrUrl):
        self.reqMgrUrl = reqMgrUrl
        self.restSender = JSONRequests(reqMgrUrl)
        d = dict(endpoint = self.reqMgrUrl)
        self.reqMgrService = RequestManager(d)
        

    def queryAllRequests(self):
        """
        Returns all requests stored at ReqMgr instance.
        
        """
        logging.info("Querying all requests at ReqMgr instance ...")
        r = self.reqMgrService.getRequestNames()
        print "Found %s requests:" % len(r)
        for req in r:
            print req
        

    def queryRequest(self, requestName):
        """
        Query a specific request according to the input argument.
        
        """
        urlQuery = "request/%s" % requestName
        logging.info("Querying request '%s'" % requestName)
        logging.info("Query: '%s':" % urlQuery)
        r = self.restSender.get(urlQuery)
        print str(r)
        
        
    def createRequests(self, numRequests):
        """
        Inject new numRequests into ReqMgr instance.
        (see ReqMgr_t testE how to create a request)
        
        """
        logging.info("Creating %s new requests ..." % numRequests)
        schema = ReReco.getTestArguments()
        schema['RequestName'] = 'TestReReco'
        schema['RequestType'] = 'ReReco'
        schema['CmsPath'] = "/uscmst1/prod/sw/cms"
        schema['Requestor'] = '%s' % "zmaxa"
        schema['Group'] = '%s' % "DATAOPS"
        schema['BlockWhitelist'] = ['/dataset/dataset/dataset#alpha']
        schema['BlockBlacklist'] = ['/dataset/dataset/dataset#beta']
        schema['Campaign'] = 'MyTestCampaign'        
        for i in range(numRequests):
            urlQuery = "request/testRequest"
            print "Query: '%s':" % urlQuery
            print "Schema (request): '%s'" % schema
            r = self.restSender.put(urlQuery, schema)
            # print "request creating response: ", r
            print "created: ", r[0]["RequestName"]  
                        
        
    def deleteRequest(self, requestNames):
        """
        Delete requests specified in the input, more request names
        are comma-separated.
        
        """
        logging.info("Deleting requests ...")
        for reqName in requestNames.split(','):
            reqName = reqName.strip()
            urlQuery = "request/%s" % reqName
            logging.info("Deleting request (request_name): '%s'" % reqName)
            logging.info("Query: '%s':" % urlQuery)
            r = self.restSender.delete(urlQuery)
            
            
    def requestChangeStates(self, reqName):
        """
        Route the request (spec. by the request name) in the input
        through a series of possible request states.
        
        """
        logging.info("Changing state of request %s ..." % reqName)
        def changeState(requestName, urlQuery):
            logging.info("Query: '%s' ..." % urlQuery)
            r = self.restSender.put(urlQuery)
            r = self.restSender.get("request/%s" % requestName)
            #assert r[0]["RequestStatus"] == statusName
            logging.info("Querying modified request, new state: %s" % r[0]["RequestStatus"])

        # once a request is created, it's in 'new' state
        # states transition has to be an allowed one as defined here:
        # WMCore/RequestManager/RequestDB/Settings/RequestStatus.py
        statesQueries = ["request/%s?status=%s" % (reqName, "testing-approved"),
                         "request/%s?status=%s" % (reqName, "testing"),
                         "request/%s?status=%s" % (reqName, "tested"),
                         "request/%s?status=%s" % (reqName, "assignment-approved"),
                         # create an assignment now
                         # need quote because of space in the team name
                         # (previous name - White Sox)
                         urllib.quote("assignment/%s/%s" % (TEAM_NAME, reqName))]
        for query in statesQueries:
            changeState(reqName, query)
            
                    
    def setup(self):
        """
        Setup ReqMgr instance for dealing with requests - needs to create
        a user, group, SW releases entries, etc.
        as done in test/python/WMCore_t/RequestManager_t/ReqMgr_t.py
        
        """
        logging.info("ReqMgr setup ...")
        def doQuery(urlQuery):
            logging.info("Query: '%s' ..." % urlQuery)
            r = None
            try:
                r = self.restSender.put(urlQuery)
            except Exception as ex:
                print "exception"
                print str(ex) 
            print "response:", r
        queries = ["user/[email protected]",
                   "group/DATAOPS",
                   "group/DATAOPS/zmaxa",
                   urllib.quote("team/" + TEAM_NAME),
                   "version/%s" % "CMSSW_3_5_8"]
        for q in queries:
            doQuery(q)
        logging.info("ReqMgr setup finished, listing known users ...")
        q = "user/"
        r = self.restSender.get(q)
        print r
Пример #39
0
class CleanCouchPoller(BaseWorkerThread):
    """
    Cleans up local couch db according the the given condition.
    1. Cleans local couch db when request is completed and reported to cental db.
       This will clean up local couchdb, local summary db, local queue
       
    2. Cleans old couchdoc which is created older than the time threshold
    
    """
    def __init__(self, config):
        """
        Initialize config
        """
        BaseWorkerThread.__init__(self)
        # set the workqueue service for REST call
        self.config = config
        
    def setup(self, parameters):
        """
        Called at startup
        """
        self.teamName = self.config.Agent.teamName
        # set the connection for local couchDB call
        self.useReqMgrForCompletionCheck = getattr(self.config.TaskArchiver, 'useReqMgrForCompletionCheck', True)
        self.archiveDelayHours = getattr(self.config.TaskArchiver, 'archiveDelayHours', 0)
        self.wmstatsCouchDB = WMStatsWriter(self.config.TaskArchiver.localWMStatsURL, appName="WMStatsAgent")
        
        #TODO: we might need to use local db for Tier0
        self.centralRequestDBReader = RequestDBReader(self.config.AnalyticsDataCollector.centralRequestDBURL, 
                                                      couchapp=self.config.AnalyticsDataCollector.RequestCouchApp)
        
        if self.useReqMgrForCompletionCheck:
            self.deletableState = "announced"
            self.centralRequestDBWriter = RequestDBWriter(self.config.AnalyticsDataCollector.centralRequestDBURL, 
                                                          couchapp=self.config.AnalyticsDataCollector.RequestCouchApp)
            self.reqmgr2Svc = ReqMgr(self.config.TaskArchiver.ReqMgr2ServiceURL)
            #TODO: remove this when reqmgr2 replace reqmgr completely (reqmgr2Only)
            self.reqmgrSvc = RequestManager({'endpoint': self.config.TaskArchiver.ReqMgrServiceURL})
        else:
            # Tier0 case
            self.deletableState = "completed"
            # use local for update
            self.centralRequestDBWriter = RequestDBWriter(self.config.AnalyticsDataCollector.localT0RequestDBURL, 
                                                          couchapp=self.config.AnalyticsDataCollector.RequestCouchApp)
        
        jobDBurl = sanitizeURL(self.config.JobStateMachine.couchurl)['url']
        jobDBName = self.config.JobStateMachine.couchDBName
        self.jobCouchdb  = CouchServer(jobDBurl)
        self.jobsdatabase = self.jobCouchdb.connectDatabase("%s/jobs" % jobDBName)
        self.fwjrdatabase = self.jobCouchdb.connectDatabase("%s/fwjrs" % jobDBName)
        
        statSummaryDBName = self.config.JobStateMachine.summaryStatsDBName
        self.statsumdatabase = self.jobCouchdb.connectDatabase(statSummaryDBName)

    def algorithm(self, parameters):
        """
        Get information from wmbs, workqueue and local couch.
          - It deletes old wmstats docs
          - Archive workflows
        """
        try:
            logging.info("Cleaning up the old request docs")
            report = self.wmstatsCouchDB.deleteOldDocs(self.config.TaskArchiver.DataKeepDays)
            logging.info("%s docs deleted" % report)
            
            logging.info("Cleaning up the archived request docs")
            report = self.cleanAlreadyArchivedWorkflows()
            logging.info("%s archived workflows deleted" % report)
            
            # archiving only workflows that I own (same team)
            logging.info("Getting requests in '%s' state for team '%s'", self.deletableState,
                                                                           self.teamName)
            endTime = int(time.time()) - self.archiveDelayHours * 3600
            wfs = self.centralRequestDBReader.getRequestByTeamAndStatus(self.teamName,
                                                                        self.deletableState)
            commonWfs = self.centralRequestDBReader.getRequestByStatusAndStartTime(self.deletableState, 
                                                                                   False, endTime)
            deletableWorkflows = list(set(wfs) & set(commonWfs))
            logging.info("Ready to archive normal %s workflows", len(deletableWorkflows))
            numUpdated = self.archiveWorkflows(deletableWorkflows, "normal-archived")
            logging.info("archive normal %s workflows", numUpdated)
            
            abortedWorkflows = self.centralRequestDBReader.getRequestByStatus(["aborted-completed"])
            logging.info("Ready to archive aborted %s workflows", len(abortedWorkflows))
            numUpdated = self.archiveWorkflows(abortedWorkflows, "aborted-archived")
            logging.info("archive aborted %s workflows", numUpdated)
            
            rejectedWorkflows = self.centralRequestDBReader.getRequestByStatus(["rejected"])
            logging.info("Ready to archive rejected %s workflows", len(rejectedWorkflows))
            numUpdated = self.archiveWorkflows(rejectedWorkflows, "rejected-archived")
            logging.info("archive rejected %s workflows", numUpdated)

        except Exception as ex:
            logging.error(str(ex))
            logging.error("Error occurred, will try again next cycle")
    
    def archiveWorkflows(self, workflows, archiveState):
        updated = 0
        for workflowName in workflows:
            if self.cleanAllLocalCouchDB(workflowName):
                if self.useReqMgrForCompletionCheck:
                    try:
                        #TODO: try reqmgr1 call if it fails (reqmgr2Only - remove this line when reqmgr is replaced)
                        self.reqmgrSvc.updateRequestStatus(workflowName, archiveState)
                        #And replace with this - remove all the excption
                        #self.reqmgr2Svc.updateRequestStatus(workflowName, archiveState)
                    except HTTPException as ex:
                        # If we get an HTTPException of 404 means reqmgr2 request
                        if ex.status == 404:
                            # try reqmgr2 call
                            msg = "%s : reqmgr2 request: %s" % (workflowName, str(ex))
                            logging.warning(msg)
                            self.reqmgr2Svc.updateRequestStatus(workflowName, archiveState)
                        else:
                            msg = "%s : fail to update status with HTTP error: %s" % (workflowName, str(ex))
                            logging.error(msg)
                            raise ex
                            
                    updated += 1 
                    logging.debug("status updated to %s %s",  archiveState, workflowName)
                else:
                    # tier0 update case
                    self.centralRequestDBWriter.updateRequestStatus(workflowName, archiveState)
        return updated
    
    def deleteWorkflowFromJobCouch(self, workflowName, db):
        """
        _deleteWorkflowFromCouch_

        If we are asked to delete the workflow from couch, delete it
        to clear up some space.

        Load the document IDs and revisions out of couch by workflowName,
        then order a delete on them.
        """
        options = {"startkey": [workflowName], "endkey": [workflowName, {}], "reduce": False}
        
        if db == "JobDump":
            couchDB = self.jobsdatabase
            view = "jobsByWorkflowName"
        elif db == "FWJRDump":
            couchDB = self.fwjrdatabase
            view = "fwjrsByWorkflowName"
        elif db == "SummaryStats":
            couchDB = self.statsumdatabase
            view = None
        elif db == "WMStatsAgent":
            couchDB = self.wmstatsCouchDB.getDBInstance()
            view = "allWorkflows"
            options = {"key": workflowName, "reduce": False}
            
        if view == None:
            try:
                committed = couchDB.delete_doc(workflowName)
            except CouchNotFoundError as ex:
                return {'status': 'warning', 'message': "%s: %s" % (workflowName, str(ex))}
        else:
            try:
                jobs = couchDB.loadView(db, view, options = options)['rows']
            except Exception as ex:
                errorMsg = "Error on loading jobs for %s" % workflowName
                logging.warning("%s/n%s" % (str(ex), errorMsg))
                return {'status': 'error', 'message': errorMsg}
            
            for j in jobs:
                doc = {}
                doc["_id"]  = j['value']['id']
                doc["_rev"] = j['value']['rev']
                couchDB.queueDelete(doc)
            committed = couchDB.commit()
        
        if committed:
            #create the error report
            errorReport = {}
            deleted = 0
            status = "ok"
            for data in committed:
                if 'error' in data:
                    errorReport.setdefault(data['error'], 0)
                    errorReport[data['error']] += 1
                    status = "error"
                else:
                    deleted += 1
            return {'status': status, 'delete': deleted, 'message': errorReport}
        else:
            return {'status': 'warning', 'message': "no %s exist" % workflowName}


    def cleanAllLocalCouchDB(self, workflowName):
        logging.info("Deleting %s from JobCouch" % workflowName)
        
        jobReport = self.deleteWorkflowFromJobCouch(workflowName, "JobDump")
        logging.debug("%s docs deleted from JobDump", jobReport)
        
        fwjrReport = self.deleteWorkflowFromJobCouch(workflowName, "FWJRDump")
        logging.debug("%s docs deleted from FWJRDump", fwjrReport)
        
        summaryReport = self.deleteWorkflowFromJobCouch(workflowName, "SummaryStats")
        logging.debug("%s docs deleted from SummaryStats", summaryReport)
        
        wmstatsReport = self.deleteWorkflowFromJobCouch(workflowName, "WMStatsAgent")
        logging.debug("%s docs deleted from wmagent_summary", wmstatsReport)
        
        # if one of the procedure fails return False
        if (jobReport["status"] == "error" or fwjrReport["status"] == "error" or 
            wmstatsReport["status"] == "error"):
            return False
        # other wise return True.
        return True
        
    def cleanAlreadyArchivedWorkflows(self):
        """
        loop through the workflows in couchdb, if archived delete all the data in couchdb
        """
        
        numDeletedRequests = 0
        try:
            localWMStats = self.wmstatsCouchDB.getDBInstance()
            options = {"group_level": 1, "reduce": True}
            
            results = localWMStats.loadView("WMStatsAgent", "allWorkflows", options = options)['rows']
            requestNames = [x['key'] for x in results]
            logging.info("There are %s workfows to check for archived status" % len(requestNames))
            
            workflowDict = self.centralRequestDBReader.getStatusAndTypeByRequest(requestNames)
            
            for request, value in workflowDict.items():
                if value[0].endswith("-archived"):
                    self.cleanAllLocalCouchDB(request)
                    numDeletedRequests += 1
        
        except Exception as ex:
            errorMsg = "Error on loading workflow list from wmagent_summary db"
            logging.warning("%s/n%s" % (errorMsg, str(ex)))
            
        return numDeletedRequests
Пример #40
0
class ReqMgrTester(object):
    def __init__(self, reqMgrUrl):
        self.reqMgrUrl = reqMgrUrl
        self.restSender = JSONRequests(reqMgrUrl)
        d = dict(endpoint = self.reqMgrUrl)
        self.reqMgrService = RequestManager(d)
        

    def queryAllRequests(self):
        """
        Returns all requests stored at ReqMgr instance.
        
        """
        logging.info("Querying all requests at ReqMgr instance ...")
        r = self.reqMgrService.getRequestNames()
        print "Found %s requests:" % len(r)
        for req in r:
            print req
        

    def queryRequest(self, requestName):
        """
        Query a specific request according to the input argument.
        
        """
        urlQuery = "request/%s" % requestName
        logging.info("Querying request '%s'" % requestName)
        logging.info("Query: '%s':" % urlQuery)
        r = self.restSender.get(urlQuery)
        print str(r)
        
        
    def createRequests(self, numRequests):
        """
        Inject new numRequests into ReqMgr instance.
        (see ReqMgr_t testE how to create a request)
        
        """
        logging.info("Creating %s new requests ..." % numRequests)
        schema = ReReco.getTestArguments()
        schema['RequestName'] = 'TestReReco'
        schema['RequestType'] = 'ReReco'
        schema['CmsPath'] = "/uscmst1/prod/sw/cms"
        schema['Requestor'] = '%s' % "testinguser"
        schema['Group'] = '%s' % "PeopleLikeMe"
        schema['BlockWhitelist'] = ['/dataset/dataset/dataset#alpha']
        schema['BlockBlacklist'] = ['/dataset/dataset/dataset#beta']
        schema['Campaign'] = 'MyTestCampaign'        
        for i in range(numRequests):
            urlQuery = "request/testRequest"
            logging.info("Query: '%s':" % urlQuery)
            r = self.restSender.put(urlQuery, schema)
            # print "request creating response: ", r
            print "created: ", r[0]["RequestName"]  
                        
        
    def deleteRequest(self, requestNames):
        """
        Delete requests specified in the input, more request names
        are comma-separated.
        
        """
        logging.info("Deleting requests ...")
        for reqName in requestNames.split(','):
            reqName = reqName.strip()
            urlQuery = "request/%s" % reqName
            logging.info("Deleting request (request_name): '%s'" % reqName)
            logging.info("Query: '%s':" % urlQuery)
            r = self.restSender.delete(urlQuery)

            
    def injectOpsClipboard(self, reqName, couchUrl, couchDbName):
        """
        Once a request reaches "ops-hold" state, it can be injected into
        CouchDB, application OpsClipboard, for further manipulation.
        Do this here with the reqName request.
        
        OpsClipboard.inject() method which does in the CouchDB injection
        is called from WMCore/HTTPFrontEnd/RequestManager/Assign.py
            handleAssignmentPage method (which currently, 2012-01, doesn't
            have any unittest nor REST API) (used only from the Assignment webpage)
            
        Works when running locally accessing CouchDB behind frontend:
        py test/data/ReqMgr/reqmgr-load_example_data.py -u https://localhost:2000/reqmgr/reqMgr/ \
            -t testinguser_120131_213320_2161 -i -o https://localhost:2000/couchdb/ \
            -a ops_clipboard        
        
        """
        # find out campaign name associated with this request
        r = self.restSender.get("request/%s" % reqName)
        campaign = r[0]["Campaign"]
        logging.info("Campaign: %s" % campaign)
        requests = [{u"RequestName": reqName, u"CampaignName": campaign}]
        OpsClipboard.inject(couchUrl, couchDbName, *requests)
            
            
    def requestChangeStates(self, reqName, injectOpsClipboard, couchUrl, couchDbName):
        """
        Route the request (spec. by the request name) in the input
        through a series of possible request states.
        
        """
        logging.info("Changing state of request %s ..." % reqName)
        def changeState(requestName, urlQuery):
            logging.info("Query: '%s' ..." % urlQuery)
            r = self.restSender.put(urlQuery)
            r = self.restSender.get("request/%s" % requestName)
            #assert r[0]["RequestStatus"] == statusName
            logging.info("Querying modified request, new state: %s" % r[0]["RequestStatus"])

        # once a request is created, it's in 'new' state
        # states transition has to be an allowed one as defined here:
        # WMCore/RequestManager/RequestDB/Settings/RequestStatus.py
        statesQueries = ["request/%s?status=%s" % (reqName, "testing-approved"),
                         "request/%s?status=%s" % (reqName, "testing"),
                         "request/%s?status=%s" % (reqName, "tested"),
                         "request/%s?status=%s" % (reqName, "assignment-approved"),
                         # create an assignment now
                         # need quote because of space in the team name
                         urllib.quote("assignment/%s/%s" % (TEAM_NAME, reqName)),
                         "request/%s?status=%s" % (reqName, "ops-hold")]                 
        for query in statesQueries:
            changeState(reqName, query)
            
        if injectOpsClipboard:
            self.injectOpsClipboard(reqName, couchUrl, couchDbName)
                
        
    def setup(self):
        """
        Setup ReqMgr instance for dealing with requests - needs to create
        a user, group, SW releases entries, etc.
        as done in test/python/WMCore_t/RequestManager_t/ReqMgr_t.py
        
        """
        logging.info("ReqMgr setup ...")
        def doQuery(urlQuery):
            logging.info("Query: '%s' ..." % urlQuery)
            r = None
            try:
                r = self.restSender.put(urlQuery)
            except Exception as ex:
                print "exception"
                print str(ex) 
            print "response:", r
        queries = ["user/[email protected]",
                   "group/PeopleLikeMe",
                   "group/PeopleLikeMe/testinguser",
                   urllib.quote("team/" + TEAM_NAME),
                   "version/%s" % "CMSSW_3_5_8"]
        for q in queries:
            doQuery(q)
        logging.info("ReqMgr setup finished, listing known users ...")
        q = "user/"
        r = self.restSender.get(q)
        print r
Пример #41
0
class JobUpdaterPoller(BaseWorkerThread):
    """
    _JobUpdaterPoller_

    Poller class for the JobUpdater
    """

    def __init__(self, config):
        """
        __init__
        """
        BaseWorkerThread.__init__(self)
        self.config = config

        self.bossAir = BossAirAPI(config = self.config)
        self.reqmgr = RequestManager({'endpoint' : self.config.JobUpdater.reqMgrUrl})
        self.workqueue = WorkQueue(self.config.WorkQueueManager.couchurl,
                                   self.config.WorkQueueManager.dbname)

        myThread = threading.currentThread()

        self.daoFactory = DAOFactory(package = "WMCore.WMBS",
                                     logger = myThread.logger,
                                     dbinterface = myThread.dbi)

        self.listWorkflowsDAO = self.daoFactory(classname = "Workflow.ListForJobUpdater")
        self.updateWorkflowPrioDAO = self.daoFactory(classname = "Workflow.UpdatePriority")
        self.executingJobsDAO = self.daoFactory(classname = "Jobs.GetNumberOfJobsForWorkflowTaskStatus")


    def setup(self, parameters = None):
        """
        _setup_
        """
        pass

    def terminate(self, parameters = None):
        """
        _terminate_

        Terminate gracefully.
        """
        pass

    def algorithm(self, parameters = None):
        """
        _algorithm_
        """
        logging.info("Synchronizing priorities with ReqMgr...")
        self.synchronizeJobPriority()

    def synchronizeJobPriority(self):
        """
        _synchronizeJobPriority_

        Check WMBS and WorkQueue for active workflows and compare with the
        ReqMgr for priority changes. If a priority change occurs
        then update the job priority in the batch system and
        the elements in the local queue that have not been injected yet.
        """
        # Update the priority of workflows that are not in WMBS and just in local queue
        priorityCache = {}
        workflowsToUpdate = {}
        workflowsToCheck = [x for x in self.workqueue.getAvailableWorkflows()]
        for workflow, priority in workflowsToCheck:
            if workflow not in priorityCache:
                try:
                    priorityCache[workflow] = self.reqmgr.getRequest(workflow)['RequestPriority']
                except Exception, ex:
                    logging.error("Couldn't retrieve the priority of request %s" % workflow)
                    logging.error("Error: %s" % ex)
                    continue
            if priority != priorityCache[workflow]:
                workflowsToUpdate[workflow] = priorityCache[workflow]
        for workflow in workflowsToUpdate:
            self.workqueue.updatePriority(workflow, workflowsToUpdate[workflow])

        # Check the workflows in WMBS
        priorityCache = {}
        workflowsToUpdateWMBS = {}
        workflowsToCheck = self.listWorkflowsDAO.execute()
        for workflowEntry in workflowsToCheck:
            workflow = workflowEntry['name']
            if workflow not in priorityCache:
                try:
                    priorityCache[workflow] = self.reqmgr.getRequest(workflow)['RequestPriority']
                except Exception, ex:
                    logging.error("Couldn't retrieve the priority of request %s" % workflow)
                    logging.error("Error: %s" % ex)
                    continue
            requestPriority = priorityCache[workflow]
            if requestPriority != workflowEntry['workflow_priority']:
                # Update the workqueue priority for the Available elements
                self.workqueue.updatePriority(workflow, priorityCache[workflow])
                # Check if there are executing jobs for this particular task
                if self.executingJobsDAO.execute(workflow, workflowEntry['task']) > 0:
                    self.bossAir.updateJobInformation(workflow, workflowEntry['task'],
                                                      requestPriority = priorityCache[workflow],
                                                      taskPriority = workflowEntry['task_priority'])
                workflowsToUpdateWMBS[workflow] = priorityCache[workflow]
Пример #42
0
class TaskArchiverPoller(BaseWorkerThread):
    """
    Polls for Ended jobs

    List of attributes

    requireCouch:  raise an exception on couch failure instead of ignoring
    """
    def __init__(self, config):
        """
        Initialise class members
        """
        BaseWorkerThread.__init__(self)

        myThread = threading.currentThread()
        self.daoFactory = DAOFactory(package = "WMCore.WMBS",
                                     logger = myThread.logger,
                                     dbinterface = myThread.dbi)
        
        self.dbsDaoFactory = DAOFactory(package = "WMComponent.DBS3Buffer",
                                     logger = myThread.logger, 
                                     dbinterface = myThread.dbi)

        self.config      = config
        self.jobCacheDir = self.config.JobCreator.jobCacheDir

        if getattr(self.config.TaskArchiver, "useWorkQueue", False) != False:
            # Get workqueue setup from config unless overridden
            if hasattr(self.config.TaskArchiver, 'WorkQueueParams'):
                self.workQueue = localQueue(**self.config.TaskArchiver.WorkQueueParams)
            else:
                from WMCore.WorkQueue.WorkQueueUtils import queueFromConfig
                self.workQueue = queueFromConfig(self.config)
        else:
            self.workQueue = None

        self.timeout           = getattr(self.config.TaskArchiver, "timeOut", None)
        self.useReqMgrForCompletionCheck   = getattr(self.config.TaskArchiver, 'useReqMgrForCompletionCheck', True)
        
        if not self.useReqMgrForCompletionCheck:
            #sets the local monitor summary couch db
            self.requestLocalCouchDB = RequestDBWriter(self.config.AnalyticsDataCollector.localT0RequestDBURL, 
                                                   couchapp = self.config.AnalyticsDataCollector.RequestCouchApp)
            self.centralCouchDBWriter = self.requestLocalCouchDB;
        else:
            self.centralCouchDBWriter = RequestDBWriter(self.config.AnalyticsDataCollector.centralRequestDBURL)
            
            self.reqmgr2Svc = ReqMgr(self.config.TaskArchiver.ReqMgr2ServiceURL)
            #TODO: remove this when reqmgr2 replace reqmgr completely (reqmgr2Only)
            self.reqmgrSvc = RequestManager({'endpoint': self.config.TaskArchiver.ReqMgrServiceURL})

        #Load the cleanout state ID and save it
        stateIDDAO = self.daoFactory(classname = "Jobs.GetStateID")
        self.stateID = stateIDDAO.execute("cleanout")

        return

    def terminate(self, params):
        """
        _terminate_

        This function terminates the job after a final pass
        """
        logging.debug("terminating. doing one more pass before we die")
        self.algorithm(params)
        return

    def algorithm(self, parameters = None):
        """
        _algorithm_

        Executes the two main methods of the poller:
        1. findAndMarkFinishedSubscriptions
        2. completeTasks
        Final result is that finished workflows get their summary built and uploaded to couch,
        and all traces of them are removed from the agent WMBS and couch (this last one on demand).
        """
        try:
            self.findAndMarkFinishedSubscriptions()
            (finishedwfs, finishedwfsWithLogCollectAndCleanUp) = self.getFinishedWorkflows()
            # set the data cache which can be used other thread (no ther thread should set the data cache)
            DataCache.setFinishedWorkflows(finishedwfsWithLogCollectAndCleanUp)
            self.completeTasks(finishedwfs)
        except WMException:
            myThread = threading.currentThread()
            if getattr(myThread, 'transaction', False) \
                   and getattr(myThread.transaction, 'transaction', False):
                myThread.transaction.rollback()
            raise
        except Exception as ex:
            myThread = threading.currentThread()
            msg = "Caught exception in TaskArchiver\n"
            msg += str(ex)
            if getattr(myThread, 'transaction', False) \
                   and getattr(myThread.transaction, 'transaction', False):
                myThread.transaction.rollback()
            raise TaskArchiverPollerException(msg)

        return

    def findAndMarkFinishedSubscriptions(self):
        """
        _findAndMarkFinishedSubscriptions_

        Find new finished subscriptions and mark as finished in WMBS.
        """
        myThread = threading.currentThread()

        myThread.transaction.begin()

        #Get the subscriptions that are now finished and mark them as such
        logging.info("Polling for finished subscriptions")
        finishedSubscriptions = self.daoFactory(classname = "Subscriptions.MarkNewFinishedSubscriptions")
        finishedSubscriptions.execute(self.stateID, timeOut = self.timeout)
        logging.info("Finished subscriptions updated")

        myThread.transaction.commit()

        return
    
    
    def getFinishedWorkflows(self):
        """
        1. Get finished workflows (a finished workflow is defined in Workflow.GetFinishedWorkflows)
        2. Get finished workflows with logCollect and Cleanup only.
        3. combined those and make return 
           finishedwfs - without LogCollect and CleanUp task
           finishedwfsWithLogCollectAndCleanUp - including LogCollect and CleanUp task
        """
        
        finishedWorkflowsDAO = self.daoFactory(classname = "Workflow.GetFinishedWorkflows")
        finishedwfs = finishedWorkflowsDAO.execute()
        finishedLogCollectAndCleanUpwfs = finishedWorkflowsDAO.execute(onlySecondary=True)
        finishedwfsWithLogCollectAndCleanUp = {}
        for wf in finishedLogCollectAndCleanUpwfs:
            if wf in finishedwfs:
                finishedwfsWithLogCollectAndCleanUp[wf] = finishedwfs[wf]
        return (finishedwfs, finishedwfsWithLogCollectAndCleanUp)
        
    def completeTasks(self, finishedwfs):
        """
        _completeTasks_

        This method will call several auxiliary methods to do the following:
        
        1. Notify the WorkQueue about finished subscriptions
        2. update dbsbuffer_workflow table with finished subscription
        """


        #Only delete those where the upload and notification succeeded
        logging.info("Found %d candidate workflows for completing: %s" % (len(finishedwfs),finishedwfs.keys()))
        # update the completed flag in dbsbuffer_workflow table so blocks can be closed
        # create updateDBSBufferWorkflowComplete DAO
        if len(finishedwfs) == 0:
            return
        
        completedWorkflowsDAO = self.dbsDaoFactory(classname = "UpdateWorkflowsToCompleted")
        
        centralCouchAlive = True
        try:
            #TODO: need to enable when reqmgr2 -wmstats is ready
            #abortedWorkflows = self.reqmgrCouchDBWriter.getRequestByStatus(["aborted"], format = "dict");
            abortedWorkflows = self.centralCouchDBWriter.getRequestByStatus(["aborted"])
            logging.info("There are %d requests in 'aborted' status in central couch." % len(abortedWorkflows))
            forceCompleteWorkflows = self.centralCouchDBWriter.getRequestByStatus(["force-complete"])
            logging.info("List of 'force-complete' workflows in central couch: %s" % forceCompleteWorkflows)
            
        except Exception as ex:
            centralCouchAlive = False
            logging.error("we will try again when remote couch server comes back\n%s" % str(ex))
        
        if centralCouchAlive:
            for workflow in finishedwfs:
                try:
                    #Notify the WorkQueue, if there is one
                    if self.workQueue != None:
                        subList = []
                        logging.info("Marking subscriptions as Done ...")
                        for l in finishedwfs[workflow]["workflows"].values():
                            subList.extend(l)
                        self.notifyWorkQueue(subList)
                    
                    #Now we know the workflow as a whole is gone, we can delete the information from couch
                    if not self.useReqMgrForCompletionCheck:
                        self.requestLocalCouchDB.updateRequestStatus(workflow, "completed")
                        logging.info("status updated to completed %s" % workflow)
    
                    if workflow in abortedWorkflows:
                        #TODO: remove when reqmgr2-wmstats deployed
                        newState = "aborted-completed"
                    elif workflow in forceCompleteWorkflows:
                        newState = "completed"
                    else:
                        newState = None
                        
                    if newState != None:
                        # update reqmgr workload document only request mgr is installed
                        if not self.useReqMgrForCompletionCheck:
                            # commented out untill all the agent is updated so every request have new state
                            # TODO: agent should be able to write reqmgr db diretly add the right group in
                            # reqmgr
                            self.requestLocalCouchDB.updateRequestStatus(workflow, newState)
                        else:
                            try:
                                #TODO: try reqmgr1 call if it fails (reqmgr2Only - remove this line when reqmgr is replaced)
                                logging.info("Updating status to '%s' in both oracle and couchdb ..." % newState)
                                self.reqmgrSvc.updateRequestStatus(workflow, newState)
                                #And replace with this - remove all the excption
                                #self.reqmgr2Svc.updateRequestStatus(workflow, newState)
                            except httplib.HTTPException as ex:
                                # If we get an HTTPException of 404 means reqmgr2 request
                                if ex.status == 404:
                                    # try reqmgr2 call
                                    msg = "%s : reqmgr2 request: %s" % (workflow, str(ex))
                                    logging.warning(msg)
                                    self.reqmgr2Svc.updateRequestStatus(workflow, newState)
                                else:
                                    msg = "%s : fail to update status %s  with HTTP error: %s" % (workflow, newState, str(ex))
                                    logging.error(msg)
                                    raise ex
                            
                        logging.info("status updated to '%s' : %s" % (newState, workflow))
                    
                    completedWorkflowsDAO.execute([workflow])
        
                except TaskArchiverPollerException as ex:

                    #Something didn't go well when notifying the workqueue, abort!!!
                    logging.error("Something bad happened while archiving tasks.")
                    logging.error(str(ex))
                    continue
                except Exception as ex:
                    #Something didn't go well on couch, abort!!!
                    msg = "Problem while archiving tasks for workflow %s\n" % workflow
                    msg += "Exception message: %s" % str(ex)
                    msg += "\nTraceback: %s" % traceback.format_exc()
                    logging.error(msg)
                    continue
        return
    
    def notifyWorkQueue(self, subList):
        """
        _notifyWorkQueue_

        Tells the workQueue component that a particular subscription,
        or set of subscriptions, is done.  Receives confirmation
        """

        for sub in subList:
            try:
                self.workQueue.doneWork(SubscriptionId = sub)
            except WorkQueueNoMatchingElements:
                #Subscription wasn't known to WorkQueue, feel free to clean up
                logging.info("Local WorkQueue knows nothing about this subscription: %s" % sub)
                pass
            except Exception as ex:
                msg = "Error talking to workqueue: %s\n" % str(ex)
                msg += "Tried to complete the following: %s\n" % sub
                raise TaskArchiverPollerException(msg)

        return
Пример #43
0
class ReqMgrTest(RESTBaseUnitTest):
    """
    Basic test for the ReqMgr services.
    Setup is done off-screen in RESTBaseUnitTest - this makes
    things confusing
    
    """

    def setUp(self):
        """
        setUP global values
        Database setUp is done in base class
        """
        self.couchDBName = "reqmgr_t_0"
        RESTBaseUnitTest.setUp(self)
        self.testInit.setupCouch("%s" % self.couchDBName, "ConfigCache", "ReqMgr")
        self.testInit.setupCouch("%s_wmstats" % self.couchDBName,
                                 "WMStats")
        self.testInit.setupCouch("%s_acdc" % self.couchDBName,
                                 "ACDC", "GroupUser")
        reqMgrHost = self.config.getServerUrl()
        self.jsonSender = JSONRequests(reqMgrHost)

        self.params = {}
        self.params['endpoint'] = reqMgrHost
        self.reqService = RequestManager(self.params)

    def initialize(self):
        self.config = RequestManagerConfig(
                'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel')
        self.config.setFormatter('WMCore.WebTools.RESTFormatter')
        self.config.setupRequestConfig()
        self.config.setupCouchDatabase(dbName = self.couchDBName)
        self.config.setPort(12888)
        self.schemaModules = ["WMCore.RequestManager.RequestDB"]


    def tearDown(self):
        """
        tearDown 

        Tear down everything
        
        """
        RESTBaseUnitTest.tearDown(self)
        self.testInit.tearDownCouch()


    def createConfig(self, bad = False):
        """
        _createConfig_

        Create a config of some sort that we can load out of ConfigCache
        """
        
        PSetTweak = {'process': {'outputModules_': ['ThisIsAName'],
                                 'ThisIsAName': {'dataset': {'dataTier': 'RECO',
                                                             'filterName': 'Filter'}}}}

        BadTweak  = {'process': {'outputModules_': ['ThisIsAName1', 'ThisIsAName2'],
                                 'ThisIsAName1': {'dataset': {'dataTier': 'RECO',
                                                             'filterName': 'Filter'}},
                                 'ThisIsAName2': {'dataset': {'dataTier': 'RECO',
                                                             'filterName': 'Filter'}}}}

        configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = self.couchDBName)
        configCache.createUserGroup(groupname = "testGroup", username = '******')
        if bad:
            configCache.setPSetTweaks(PSetTweak = BadTweak)
        else:
            configCache.setPSetTweaks(PSetTweak = PSetTweak)
        configCache.save()

        return configCache.getCouchID()
    

    @attr("integration")
    def testA_testBasicSetUp(self):
        """
        _testBasicSetUp_
        
        Moving the tests that were in the setUp category out of it,
        mostly because I want to make sure that they don't fail inside
        the setUp statement.
        
        """
        if 'me' in self.jsonSender.get('user')[0]:
            self.jsonSender.delete('user/me')    
        self.assertFalse('me' in self.jsonSender.get('user')[0])
        self.assertEqual(self.jsonSender.put('user/[email protected]')[1], 200)
        self.assertTrue('me' in self.jsonSender.get('user')[0])

        if 'PeopleLikeMe' in self.jsonSender.get('group')[0]:
            self.jsonSender.delete('group/PeopleLikeMe')
        self.assertFalse('PeopleLikeMe' in self.jsonSender.get('group')[0])
        self.assertEqual(self.jsonSender.put('group/PeopleLikeMe')[1], 200)
        self.assertTrue( 'PeopleLikeMe' in self.jsonSender.get('group')[0])

        self.jsonSender.put('group/PeopleLikeMe/me')
        users = self.jsonSender.get('group/PeopleLikeMe')[0]['users']
        self.assertTrue('me' in users)
        groups = self.jsonSender.get('user/me')[0]['groups']
        self.assertTrue('PeopleLikeMe' in groups)
        groups2 = self.jsonSender.get('group?user=me')[0]
        self.assertTrue('PeopleLikeMe' in groups2)

        if 'White Sox' in self.jsonSender.get('team')[0]:
            self.jsonSender.delete(urllib.quote('team/White Sox'))
        self.assertFalse('White Sox' in self.jsonSender.get('team')[0])
        self.assertEqual(self.jsonSender.put(urllib.quote('team/White Sox'))[1], 200)
        self.assertTrue('White Sox' in self.jsonSender.get('team')[0])

        # some foreign key stuff to deal with
        schema = utils.getSchema()
        version = "version/" + schema["CMSSWVersion"]
        self.assertTrue(self.jsonSender.put(version)[1] == 200)
        self.assertTrue(schema["CMSSWVersion"] in self.jsonSender.get('version')[0])
        

    @attr("integration")
    def testB_ReReco(self):
        """
        _ReReco_

        Try a basic ReReco workflow
        
        """
        schema = utils.getAndSetupSchema(self)
        schema['RequestNumEvents'] = 100
        schema['SizePerEvent'] = 101
        configID = self.createConfig()
        schema["ConfigCacheID"] = configID
        schema["CouchDBName"] = self.couchDBName
        schema["CouchURL"]    = os.environ.get("COUCHURL")

        self.doRequest(schema)


    def doRequest(self, schema):
        """
        _doRequest_

        Run all tests on a basic ReReco workflow
        
        """
        requestName = schema['RequestName']
        self.assertRaises(HTTPException, self.jsonSender.delete, 'request/%s' % requestName)
        result = self.jsonSender.put('request/%s' % (requestName), schema)
        self.assertEqual(result[1], 200)
        requestName = result[0]['RequestName']

        self.assertEqual(self.jsonSender.get('request/%s' % requestName)[0]['RequestName'], requestName)

        self.jsonSender.put('request/%s?status=assignment-approved' % requestName)
        me = self.jsonSender.get('user/me')[0]
        self.assertTrue(requestName in me['requests'])
        self.assertEqual(self.jsonSender.put('request/%s?priority=5' % requestName)[1], 200)

        request = self.jsonSender.get('request/%s' % requestName)[0]
        self.assertEqual(request['RequestPriority'], 5)

        # Check LFN Bases
        self.assertEqual(request['UnmergedLFNBase'], '/store/unmerged')
        self.assertEqual(request['MergedLFNBase'], '/store/data')

        # Check Num events
        self.assertEqual(request['RequestNumEvents'], 100)
        self.assertEqual(request['SizePerEvent'], 101)

        # only certain transitions allowed
        #self.assertEqual(self.jsonSender.put('request/%s?status=running' % requestName)[1], 400)
        self.assertRaises(HTTPException, self.jsonSender.put,'request/%s?status=running' % requestName)
        request = self.jsonSender.get('request/%s' % requestName)[0]
        self.assertEqual(request['RequestStatus'], 'assignment-approved')

        self.assertTrue(self.jsonSender.put(urllib.quote('assignment/White Sox/%s' % requestName))[1] == 200)
        requestsAndSpecs = self.jsonSender.get(urllib.quote('assignment/White Sox'))[0]
        self.assertTrue(requestName in requestsAndSpecs[0])
        workloadHelper = WMWorkloadHelper()
        workloadHelper.load(requestsAndSpecs[0][1]) 
        self.assertEqual(workloadHelper.getOwner()['Requestor'], "me")
        self.assertEqual(self.jsonSender.get('assignment?request=%s'% requestName)[0], ['White Sox'])
        self.assertEqual(self.jsonSender.get('request/%s' % requestName)[0]['teams'], ['White Sox'])

        agentUrl = 'http://cmssrv96.fnal.gov/workqueue'
        self.jsonSender.put('workQueue/%s?url=%s'% (requestName, urllib.quote(agentUrl)) )
        self.assertEqual(self.jsonSender.get('workQueue/%s' % requestName)[0][0], agentUrl)
        request = self.jsonSender.get('request/%s' % requestName)[0]
        self.assertEqual(request['RequestStatus'], 'acquired')

        self.jsonSender.post('request/%s?events_written=10&files_merged=1' % requestName)
        self.jsonSender.post('request/%s?events_written=20&files_merged=2&percent_success=99.9' % requestName)
        request = self.jsonSender.get('request/%s' % requestName)[0]
        self.assertEqual(len(request['RequestUpdates']), 2)
        self.assertEqual(request['RequestUpdates'][0]['files_merged'], 1)
        self.assertEqual(request['RequestUpdates'][1]['events_written'], 20)
        self.assertEqual(request['RequestUpdates'][1]['percent_success'], 99.9)

        message = "The sheriff is near"
        jsonMessage = json.dumps(message)
        self.jsonSender.put('message/%s' % requestName, message)
        messages = self.jsonSender.get('message/%s' % requestName)
        #self.assertEqual(messages[0][0][0], message)
        for status in ['running-open', 'running-closed', 'completed']:
            self.jsonSender.put('request/%s?status=%s' % (requestName, status))

        # campaign
        self.jsonSender.put('campaign/%s' % 'TestCampaign')
        campaigns = self.jsonSender.get('campaign')[0]
        self.assertTrue('TestCampaign' in campaigns.keys())
        self.jsonSender.put('campaign/%s/%s' % ('TestCampaign', requestName))
        requestsInCampaign = self.jsonSender.get('campaign/%s' % 'TestCampaign')[0]
        self.assertTrue(requestName in requestsInCampaign.keys())

        req = self.jsonSender.get('request/%s' % requestName)[0]
        self.assertEqual(req['Campaign'], 'TestCampaign')
        self.jsonSender.delete('request/%s' % requestName)
        

    @attr("integration")
    def testC_404Errors(self):
        """
        _404Errors_

        Do some things that generate 404 errors.  This
        should be limited to requests for objects that do not
        exist.
        """
        badName = 'ThereIsNoWayThisNameShouldExist'

        # First, try to find a non-existent request
        # This should throw a 404 error.
        # The request name should not be in it
        self.checkForError(cls = 'request', badName = badName, exitCode = 404,
                           message = 'Given requestName not found')

        # Now look for non-existent user
        self.checkForError(cls = 'user', badName = badName, exitCode = 404,
                           message = 'Cannot find user')

        # Now try non-existent campaign
        self.checkForError(cls = 'campaign', badName = badName, exitCode = 404,
                           message = "Cannot find campaign")

        # Now try invalid message
        # This raises a requestName error because it searches for the request
        self.checkForError(cls = 'message', badName = badName, exitCode = 404,
                           message = "Given requestName not found", testEmpty = False)

        # Check for assignments (no teams or requests)
        # This raises a team error because it tries to load teams out first
        self.checkForError(cls = 'assignment', badName = badName, exitCode = 404,
                           message = 'Cannot find team')


    @attr("integration")
    def testD_400Errors(self):
        """
        _400Errors_

        These are failures created by invalid input, such as sending
        args to a request when it doesn't accept any.  They should
        generatore 400 Errors
        
        """
        badName = 'ThereIsNoWayThisNameShouldExist'
        
        # Attempt to send arguments to a function that doesn't accept them.
        self.checkForError(cls = 'team', badName = badName, exitCode = 400,
                           message = "Invalid input: Arguments added where none allowed")

        # Recheck for versions
        self.checkForError(cls = 'version', badName = badName, exitCode = 400,
                           message = "Invalid input: Arguments added where none allowed")

        # Break the validation
        self.checkForError(cls = 'user', badName = '!', exitCode = 400,
                           message = 'Invalid input: Input data failed validation')
        

    def checkForError(self, cls, badName, exitCode, message, testEmpty = True):
        """
        _checkForError_

        Generic function for checking for errors in JSON commands

        Does a basic check on type cls searching for name badName which hopefull
        does not exist.

        Checks to make sure that it exits with code exitCode, and that
        the error contains the string message.

        Also checks to make sure that name badName is NOT in the output

        testEmpty for those that don't handle calls to the main (i.e., who require
        an argument)
        
        """
        raises = False
        # First assert that the test to be tested is empty
        if testEmpty:
            result = self.jsonSender.get(cls)
            self.assertTrue(type(result[0]) in [type([]), type({})])
        # Next, test
        try:
            result = self.jsonSender.get('%s/%s' % (cls, badName))
        except HTTPException as ex:
            raises = True
            self.assertEqual(ex.status, exitCode)
            self.assertTrue(message in ex.result)
            self.assertFalse(badName in ex.result)
        self.assertTrue(raises)


    @attr("integration")
    def testE_CheckStatusChanges(self):
        """
        _CheckStatusChanges_
        
        Check status changes for a single request.  See whether
        we can move the request through the proper chain.  Figure
        out what happens when we fail.
        
        """
        myThread = threading.currentThread()

        userName     = '******'
        groupName    = 'Li'
        teamName     = 'Tang'
        schema       = utils.getAndSetupSchema(self,
                                               userName = userName,
                                               groupName = groupName,
                                               teamName = teamName)
        configID = self.createConfig()
        schema["ConfigCacheID"] = configID
        schema["CouchDBName"] = self.couchDBName
        schema["CouchURL"]    = os.environ.get("COUCHURL")
        result = self.jsonSender.put('request/testRequest', schema)
        self.assertEqual(result[1], 200)
        requestName = result[0]['RequestName']

        # There should only be one request in the DB
        result = GetRequest.requestID(requestName = requestName)
        self.assertEqual(result, 1)
        result = self.jsonSender.get('request/%s' % requestName)
        self.assertEqual(result[0]['Group'], groupName)
        self.assertEqual(result[0]['Requestor'], userName)

        # Let's see what we can do in terms of setting status
        self.changeStatusAndCheck(requestName = requestName,
                                  statusName  = 'new')

        # Let's try an illegal status change, just for the hell of it
        raises = False
        try:
            self.jsonSender.put('request/%s?status=assigned' % requestName)
        except HTTPException as ex:
            raises = True
            self.assertEqual(ex.status, 403)
            self.assertTrue('Failed to change status' in ex.result)
            self.assertFalse(requestName in ex.result)
        self.assertTrue(raises)

        # Now, let's try a totally bogus status
        raises = False
        try:
            self.jsonSender.put('request/%s?status=bogus' % requestName)
        except HTTPException as ex:
            raises = True
            self.assertEqual(ex.status, 403)
            self.assertTrue('Failed to change status' in ex.result)
            self.assertFalse(requestName in ex.result)
        self.assertTrue(raises)

        # We should still be in new
        result = self.jsonSender.get('request/%s' % requestName)
        self.assertEqual(result[0]['RequestStatus'], 'new')

        # Let's go on in a full loop
        self.changeStatusAndCheck(requestName = requestName,
                                  statusName  = 'testing-approved')
        self.changeStatusAndCheck(requestName = requestName,
                                  statusName  = 'testing')
        self.changeStatusAndCheck(requestName = requestName,
                                  statusName  = 'tested')
        self.changeStatusAndCheck(requestName = requestName,
                                  statusName  = 'assignment-approved')

        # This should fail, as you cannot assign a request without a team
        raises = False
        try:
            self.changeStatusAndCheck(requestName = requestName,
                                      statusName  = 'assigned')
        except HTTPException as ex:
            raises = True
            self.assertTrue('Cannot change status without a team' in ex.result)
        self.assertTrue(raises)

        self.jsonSender.put(urllib.quote('assignment/%s/%s' % (teamName, requestName)))
        self.changeStatusAndCheck(requestName = requestName,
                                  statusName  = 'acquired')
        self.changeStatusAndCheck(requestName = requestName,
                                  statusName  = 'running-open')
        self.changeStatusAndCheck(requestName = requestName,
                                  statusName  = 'running-closed')
        self.changeStatusAndCheck(requestName = requestName,
                                  statusName  = 'completed')
        self.changeStatusAndCheck(requestName = requestName,
                                  statusName  = 'closed-out')


    def changeStatusAndCheck(self, requestName, statusName):
        """
        _changeStatusAndCheck_

        Change the status of a request and make sure that
        the request actually did it.
        
        """
        self.jsonSender.put('request/%s?status=%s' % (requestName, statusName))
        result = self.jsonSender.get('request/%s' % requestName)
        self.assertEqual(result[0]['RequestStatus'], statusName)
        

    def loadWorkload(self, requestName):
        """
        _loadWorkload_

        Load the workload from couch after we've saved it there.
        
        """
        workload = WMWorkloadHelper()
        url      = '%s/%s/%s/spec' % (os.environ['COUCHURL'], self.couchDBName,
                                      requestName)
        workload.load(url)
        return workload


    def testF_TestWhitelistBlacklist(self):
        """
        _TestWhitelistBlacklist_

        Test whether or not we can assign the block/run blacklist/whitelist
        
        """
        userName     = '******'
        groupName    = 'Li'
        teamName     = 'Tang'
        schema       = utils.getAndSetupSchema(self,
                                               userName = userName,
                                               groupName = groupName,
                                               teamName = teamName)
        schema['RunWhitelist'] = [1, 2, 3]
        schema['RunBlacklist'] = [4, 5, 6]
        schema['BlockWhitelist'] = ['/dataset/dataset/dataset#alpha']
        schema['BlockBlacklist'] = ['/dataset/dataset/dataset#beta']
        configID = self.createConfig()
        schema["ConfigCacheID"] = configID
        schema["CouchDBName"] = self.couchDBName
        schema["CouchURL"]    = os.environ.get("COUCHURL")

        result = self.jsonSender.put('request/testRequest', schema)
        self.assertEqual(result[1], 200)
        requestName = result[0]['RequestName']

        workload = self.loadWorkload(requestName = requestName)
        self.assertEqual(workload.data.tasks.DataProcessing.input.dataset.runs.whitelist, schema['RunWhitelist'])
        self.assertEqual(workload.data.tasks.DataProcessing.input.dataset.runs.blacklist, schema['RunBlacklist'])
        self.assertEqual(workload.data.tasks.DataProcessing.input.dataset.blocks.whitelist, schema['BlockWhitelist'])
        self.assertEqual(workload.data.tasks.DataProcessing.input.dataset.blocks.blacklist, schema['BlockBlacklist'])

        req = self.jsonSender.get('request/%s' % requestName)
        self.assertTrue('Site Blacklist' in req[0])
        self.assertTrue('Site Whitelist' in req[0])

        schema['BlockBlacklist'] = {'1': '/dataset/dataset/dataset#beta'}

        try:
            raises = False
            result = self.jsonSender.put('request/testRequest', schema)
        except HTTPException as ex:
            raises = True
            self.assertEqual(ex.status, 400)
            print(ex.result)
            self.assertTrue("Error in Workload Validation: Argument BlockBlacklist type is incorrect in schema." in ex.result)
            pass
        self.assertTrue(raises)

        schema['BlockBlacklist'] = ['/dataset/dataset/dataset#beta']
        schema['RunWhitelist']   = {'1': '/dataset/dataset/dataset#beta'}

        try:
            raises = False
            result = self.jsonSender.put('request/testRequest', schema)
        except HTTPException as ex:
            raises = True
            self.assertEqual(ex.status, 400)
            self.assertTrue("Error in Workload Validation: Argument RunWhitelist type is incorrect in schema." in ex.result)
            pass
        self.assertTrue(raises)

        schema['RunWhitelist'] = ['hello', 'how', 'are', 'you']
        try:
            raises = True
            result = self.jsonSender.put('request/testRequest', schema)
        except HTTPException as ex:
            raises = True
            self.assertEqual(ex.status, 400)
            self.assertTrue("Error in Workload Validation: Argument RunWhitelist doesn't pass validation." in ex.result)
            pass
        self.assertTrue(raises)


    def testG_AddDuplicateUser(self):
        """
        _AddDuplicateUser_

        Test and see if we get a sensible error when adding a duplicate user.
        
        """
        userName     = '******'
        groupName    = 'Li'
        teamName     = 'Tang'
        schema       = utils.getAndSetupSchema(self,
                                               userName = userName,
                                               groupName = groupName,
                                               teamName = teamName)
        raises = False
        try:
            self.jsonSender.put('group/%s/%s' % (groupName, userName))
        except HTTPException as ex:
            self.assertTrue("User/Group Already Linked in DB" in ex.result)
            self.assertEqual(ex.status, 400)
            raises = True
        self.assertTrue(raises)


    def testH_RemoveSoftwareVersion(self):
        """
        _RemoveSoftwareVersion_

        Remove the software version after submitting the request.  See what that does.
        
        """
        myThread = threading.currentThread()
        userName     = '******'
        groupName    = 'Li'
        teamName     = 'Tang'
        schema       = utils.getAndSetupSchema(self,
                                               userName = userName,
                                               groupName = groupName,
                                               teamName = teamName)
        configID = self.createConfig()
        schema["ConfigCacheID"] = configID
        schema["CouchDBName"] = self.couchDBName
        schema["CouchURL"]    = os.environ.get("COUCHURL")
        result = self.jsonSender.put('request/testRequest', schema)
        self.assertEqual(result[1], 200)
        requestName = result[0]['RequestName']

        req = self.jsonSender.get('request/%s' % requestName)[0]
        self.assertEqual(req['SoftwareVersions'], [schema["CMSSWVersion"]])

        # Delete software versions and make sure they're gone from the DB
        SoftwareManagement.removeSoftware(softwareName = schema["CMSSWVersion"],
                                          scramArch = schema["ScramArch"])
        versions = myThread.dbi.processData("SELECT * FROM reqmgr_software")[0].fetchall()
        self.assertEqual(versions, [])
        assocs = myThread.dbi.processData("SELECT * FROM reqmgr_software_dependency")[0].fetchall()
        self.assertEqual(assocs, [])
        
        req = self.jsonSender.get('request/%s' % requestName)[0]
        self.assertEqual(req['SoftwareVersions'], [schema["CMSSWVersion"]])
        

    def testI_CheckConfigIDs(self):
        """
        _CheckConfigIDs_

        Check to see if we can pull out the ConfigIDs by request
        
        """
        userName     = '******'
        groupName    = 'Li'
        teamName     = 'Tang'
        schema       = utils.getAndSetupSchema(self,
                                               userName = userName,
                                               groupName = groupName,
                                               teamName = teamName)
        # Set some versions
        schema['ProcessingVersion'] = '2012'
        schema['AcquisitionEra']    = 'ae2012'
        schema["PrimaryDataset"]    = "ReallyFake"
        schema["RequestNumEvents"]  = 100

        configID = self.createConfig()
        schema["CouchDBName"] = self.couchDBName
        schema["CouchURL"]    = os.environ.get("COUCHURL")
        schema["ConfigCacheID"] = configID
        schema["InputDataset"]     = '/MinimumBias/Run2010B-RelValRawSkim-v1/RAW'

        result = self.jsonSender.put('request/testRequest', schema)
        self.assertEqual(result[1], 200)
        requestName = result[0]['RequestName']

        result = self.jsonSender.get('configIDs?prim=MinimumBias&proc=Run2010B-RelValRawSkim-v1&tier=RAW')[0]
        print(result)
        self.assertTrue(requestName in result.keys())
        self.assertTrue(configID in result[requestName][0])
        
        
    def testJ_CheckRequestCloning(self):
        myThread = threading.currentThread()
        userName     = '******'
        groupName    = 'Li'
        teamName     = 'Tang'
        schema       = utils.getAndSetupSchema(self,
                                               userName = userName,
                                               groupName = groupName,
                                               teamName = teamName)
        configID = self.createConfig()
        schema["ConfigCacheID"] = configID
        schema["CouchDBName"] = self.couchDBName
        schema["CouchURL"]    = os.environ.get("COUCHURL")
        schema["AcquisitionEra"] = "NewEra"
        result = self.jsonSender.put("request", schema)
        self.assertEqual(result[1], 200)
        requestName = result[0]["RequestName"]
        acquisitionEra = result[0]["AcquisitionEra"]
        self.assertTrue(schema["AcquisitionEra"], acquisitionEra)
        # set some non-default priority
        # when cloning a request which had some non default priority,
        # the priority values were lost when creating a cloned request, the
        # default values were lost. Change it here to specifically catch this case.
        priority = 300
        result = self.jsonSender.put("request/%s?priority=%s" % (requestName, priority))        
        self.assertEqual(result[1], 200)
        # get the original request from the server, although the variable result
        # shall have the same stuff in
        response = self.jsonSender.get("request/%s" % requestName)
        origRequest = response[0]
        self.assertEqual(origRequest["AcquisitionEra"], acquisitionEra)
        # test that the priority was correctly set in the brand-new request
        self.assertEqual(origRequest["RequestPriority"], priority)
        
        # test cloning not existing request
        self.assertRaises(HTTPException, self.jsonSender.put, "clone/%s" % "NotExistingRequestName")
        # correct attempt to clone the request
        # this is the new request, it'll have different name
        result = self.jsonSender.put("clone/%s" % requestName)
        # get the cloned request from the server
        respose = self.jsonSender.get("request/%s" % result[0]["RequestName"])
        clonedRequest = respose[0]
        # these request arguments shall differ in the cloned request:
        toDiffer = ["RequestName", "RequestStatus"]
        for differ in toDiffer:
            self.assertNotEqual(origRequest[differ], clonedRequest[differ])
        # check the desired status of the cloned request
        self.assertEqual(clonedRequest["RequestStatus"], "assignment-approved",
                          "Cloned request status should be 'assignment-approved', not '%s'." %
                          clonedRequest["RequestStatus"])
        # don't care about these two (they will likely be the same in the unittest
        # since the brand new request injection as well as the cloning probably
        # happen at roughly the same time)
        toDiffer.extend(["RequestDate", "timeStamp", "RequestWorkflow"])
        for differ in toDiffer:
            del origRequest[differ]
            del clonedRequest[differ]
        # check the request dictionaries
        self.assertEqual(len(origRequest), len(clonedRequest))
        for k1, k2 in zip(sorted(origRequest.keys()), sorted(clonedRequest.keys())):
            msg = ("Request values: original: %s: %s cloned: %s: %s differ" %
                   (k1, origRequest[k1], k2, clonedRequest[k2]))
            self.assertEqual(origRequest[k1], clonedRequest[k2], msg)
            

    def testK_CheckRequestFailsInjectionForbiddenInputArg(self):
        myThread = threading.currentThread()
        userName     = '******'
        groupName    = 'Li'
        teamName     = 'Tang'
        schema       = utils.getAndSetupSchema(self,
                                               userName = userName,
                                               groupName = groupName,
                                               teamName = teamName)
        from WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel import deprecatedRequestArgs
        for deprec in deprecatedRequestArgs:
            schema = utils.getSchema(groupName=groupName, userName=userName)
            schema[deprec] = "something"
            self.assertRaises(HTTPException, self.jsonSender.put, "request", schema)

    def setupACDCDatabase(self, collectionName, taskPath,
                          user, group):
        """
        _setupACDCDatabase_

        Populate an ACDC database with bogus records
        associated to certain collection name, user and task path.
        """
        acdcServer = CouchService(url = self.testInit.couchUrl,
                                  database = "%s_acdc" % self.couchDBName)
        owner = acdcServer.newOwner(group, user)
        testCollection = CouchCollection(database = self.testInit.couchDbName,
                                          url = self.testInit.couchUrl,
                                          name = collectionName)
        testCollection.setOwner(owner)
        testFileset = CouchFileset(database = self.testInit.couchDbName,
                                    url = self.testInit.couchUrl,
                                    name = taskPath)
        testCollection.addFileset(testFileset)

        testFiles = []
        for _ in range(5):
            testFile = File(lfn = makeUUID(), size = random.randint(1024, 4096),
                            events = random.randint(1024, 4096))
            testFiles.append(testFile)

        testFileset.add(testFiles)

    def testL_CascadeCloseOutAnnnouncement(self):
        """
        _testL_CascadeCloseOutAnnouncement_

        Test the cascade closeout REST call, also
        check that when announced a request deletes all ACDC records in the system.
        """
        userName     = '******'
        groupName    = 'Li'
        teamName     = 'Tang'
        schema       = utils.getAndSetupSchema(self,
                                               userName = userName,
                                               groupName = groupName,
                                               teamName = teamName)
        configID = self.createConfig()
        schema["ConfigCacheID"] = configID
        schema["CouchDBName"] = self.couchDBName
        schema["CouchURL"]    = os.environ.get("COUCHURL")

        result = self.jsonSender.put("request", schema)[0]
        originalRequest = result['RequestName']
        self.setupACDCDatabase(originalRequest, "/%s/DataProcessing" % originalRequest,
                               result['Requestor'], result['Group'])
        depth = 2
        nReq = 3
        requests = [originalRequest]
        def createChildrenRequest(parentRequest, i, nReq):
            createdRequests = []
            resubSchema = utils.getResubmissionSchema(parentRequest, "/%s/DataProcessing" % parentRequest,
                                                      groupName, userName)
            result = self.jsonSender.put("request", resubSchema)[0]
            requestName = result['RequestName']
            self.setupACDCDatabase(requestName, "/%s/DataProcessing" % requestName, result['Requestor'], result['Group'])
            createdRequests.append(requestName)
            if i:
                for _ in range(nReq):
                    createdRequests.extend(createChildrenRequest(requestName, i - 1, nReq))
            return createdRequests
        requests.extend(createChildrenRequest(originalRequest, depth, nReq))
        for request in requests:
            self.changeStatusAndCheck(request, 'assignment-approved')
        for request in requests:
            self.jsonSender.put("assignment?team=%s&requestName=%s" % (teamName, request))
        for status in ['acquired',
                       'running-open', 'running-closed',
                       'completed']:
            for request in requests:
                self.changeStatusAndCheck(request, status)
        self.jsonSender.post('closeout?requestName=%s&cascade=True' % originalRequest)
        svc = CouchService(url = self.testInit.couchUrl,
                                  database = "%s_acdc" % self.couchDBName)

        owner = svc.newOwner(groupName, userName)
        for request in requests:
            result = self.jsonSender.get('request/%s' % request)
            self.assertEqual(result[0]['RequestStatus'], 'closed-out')
            testCollection = CouchCollection(database = self.testInit.couchDbName,
                                             url = self.testInit.couchUrl,
                                             name = request)
            testCollection.setOwner(owner)
            testCollection.populate()
            self.assertNotEqual(len(testCollection["filesets"]), 0)

        self.jsonSender.post('announce?requestName=%s&cascade=True' % originalRequest)
        for request in requests:

            result = self.jsonSender.get('request/%s' % request)
            self.assertEqual(result[0]['RequestStatus'], 'announced')
            testCollection = CouchCollection(database = self.testInit.couchDbName,
                                             url = self.testInit.couchUrl,
                                             name = request)
            testCollection.setOwner(owner)
            testCollection.populate()
            self.assertEqual(len(testCollection["filesets"]), 0)

            
    def testM_PutRequestStats(self):
        userName     = '******'
        groupName    = 'Bryant'
        teamName     = 'Lakers'
        schema       = utils.getAndSetupSchema(self,
                                               userName = userName,
                                               groupName = groupName,
                                               teamName = teamName)
        configID = self.createConfig()
        schema["ConfigCacheID"] = configID
        schema["CouchDBName"] = self.couchDBName
        schema["CouchURL"]    = os.environ.get("COUCHURL")

        result = self.jsonSender.put("request", schema)[0]
        originalRequest = result['RequestName']
        stats = {'total_jobs': 100, 'input_events': 100, 'input_lumis': 100, 'input_num_files': 100}
        result = self.reqService.putRequestStats(originalRequest, stats)
        self.assertEqual(result['RequestName'], originalRequest)
Пример #44
0
class RequestLifeCycleBase_t():

    request = None
    request_name = None
    workqueue = None
    endpoint = os.environ.get('REQMGRBASEURL', 'https://localhost:8443')
    reqmgr = RequestManager({'endpoint': endpoint + '/reqmgr/reqMgr'})
    team = 'TestTeam'
    _failure_detected = False

    @recordException
    def setUp(self):
        if self.__class__._failure_detected:
            raise nose.SkipTest
        # simple ping check - check reqmgr up
        tries = 0
        while True:
            try:
                if not self.__class__.request:
                    self.__class__.reqmgr.getTeam()
                break
            except:
                tries += 1
                if tries >= 3:
                    raise nose.SkipTest("Unable to contact reqmgr")
                time.sleep(15)

    def _configCacheId(self, label):
        """Return config cache id for given config label"""
        key, cert = self.__class__.reqmgr['requests'].getKeyCert()
        configCache = ConfigCache(self.__class__.endpoint + '/couchdb',
                                  'reqmgr_config_cache',
                                  ckey=key,
                                  cert=cert)
        try:
            configCacheId = configCache.getIDFromLabel(label)
        except:
            configCacheId = None
        if configCacheId:
            return configCacheId
        # The following will fail if FWCore.ParameterSet not in PYTHONPATH
        from PSetTweaks.WMTweak import makeTweak
        configCache.createUserGroup('test', 'test')
        configDir = os.path.join(getTestBase(), '..', '..', 'test', 'data',
                                 'configs')
        configCache.addConfig(os.path.join(configDir, label + '.py'))
        configCache.setLabel(label)
        configCache.setDescription(label)
        modPath = imp.find_module(label, [configDir])
        loadedConfig = imp.load_module(label, modPath[0], modPath[1],
                                       modPath[2])
        configCache.setPSetTweaks(
            makeTweak(loadedConfig.process).jsondictionary())
        configCache.save()
        return configCache.getIDFromLabel(label)

    def _convertLabelsToId(self, config):
        fields = [
            'ProcConfigCacheID', 'Skim1ConfigCacheID', 'StepOneConfigCacheID',
            'ConfigCacheID'
        ]
        for field in fields:
            if config.get(field):
                config[field] = self._configCacheId(config[field])
        for field in ['Task1', 'Task2', 'Task3', 'Task4']:
            if config.get(field):
                config[field] = self._convertLabelsToId(config[field])
        return config

    @attr("lifecycle")
    @recordException
    def test05InjectConfigs(self):
        """Inject configs to cache"""
        self.__class__.requestParams = self._convertLabelsToId(
            self.__class__.requestParams)

    @attr("lifecycle")
    @recordException
    def test10InjectRequest(self):
        """Can inject a request"""
        self.__class__.requestParams.setdefault('RequestString',
                                                self.__class__.__name__)
        tries = 0
        while True:
            try:
                self.__class__.request = self.__class__.reqmgr.makeRequest(
                    **self.__class__.requestParams
                )['WMCore.RequestManager.DataStructs.Request.Request']
                self.__class__.request_name = self.__class__.request[
                    'RequestName']
                break
            except:
                tries += 1
                if tries > 3:
                    raise
        self.assertTrue(self.__class__.request)
        self.assertTrue(self.__class__.request_name)
        print("Injected request %s" % self.__class__.request_name)
        self.__class__.request = self.__class__.reqmgr.getRequest(
            self.__class__.request_name)
        self.assertEqual(self.__class__.request['RequestStatus'], 'new')

    @attr("lifecycle")
    @recordException
    def test20ApproveRequest(self):
        """Approve request"""
        self.__class__.reqmgr.reportRequestStatus(self.__class__.request_name,
                                                  'assignment-approved')
        self.__class__.request = self.__class__.reqmgr.getRequest(
            self.__class__.request_name)
        self.assertEqual(self.__class__.request['RequestStatus'],
                         'assignment-approved')  #

    @attr("lifecycle")
    @recordException
    def test30AssignRequest(self):
        """Assign request"""
        self.__class__.reqmgr.assign(self.__class__.request_name,
                                     self.__class__.team,
                                     "Testing",
                                     "v1",
                                     MergedLFNBase='/store/temp',
                                     UnmergedLFNBase='/store/temp')
        self.__class__.request = self.reqmgr.getRequest(
            self.__class__.request_name)
        self.assertEqual(self.__class__.request['RequestStatus'], 'assigned')

    @attr("lifecycle")
    @recordException
    def test40WorkQueueAcquires(self):
        """WorkQueue picks up request"""
        if not self.__class__.request_name:
            raise nose.SkipTest
        start = time.time()
        while True:
            workqueue = self.reqmgr.getWorkQueue(
                request=self.__class__.request_name)
            if workqueue:
                self.__class__.workqueue = WorkQueue(workqueue[0])
                self.__class__.request = self.__class__.reqmgr.getRequest(
                    self.__class__.request_name)
                self.assertTrue(
                    self.__class__.request['RequestStatus'] in ('acquired',
                                                                'running'))
                request = [x for x in self.__class__.workqueue.getJobStatusByRequest() if \
                    x['request_name'] == self.__class__.request_name]
                if [
                        x for x in request
                        if x['status'] in ('Available', 'Negotiating',
                                           'Acquired', 'Running')
                ]:
                    break
            if start + (60 * 20) < time.time():
                raise RuntimeError('timeout waiting for workqueue to acquire')
            time.sleep(15)

    @attr("lifecycle")
    @recordException
    def test50AgentAcquires(self):
        """Elements acquired by agent"""
        # skip if request already running
        self.__class__.request = self.__class__.reqmgr.getRequest(
            self.__class__.request_name)
        if self.__class__.request['RequestStatus'] == 'running':
            raise nose.SkipTest
        start = time.time()
        while True:
            request = [x for x in self.__class__.workqueue.getJobStatusByRequest() if \
                        x['request_name'] == self.__class__.request_name]
            if [x for x in request if x['status'] in ('Acquired', 'Running')]:
                break
            if start + (60 * 20) < time.time():
                raise RuntimeError('timeout waiting for agent to acquire')
            time.sleep(15)
        self.assertTrue(
            [x for x in request if x['status'] in ('Acquired', 'Running')])

    @attr("lifecycle")
    @recordException
    def test60RequestRunning(self):
        """Request running"""
        start = time.time()
        while True:
            request = [x for x in self.__class__.workqueue.getJobStatusByRequest() if \
                    x['request_name'] == self.__class__.request_name]
            childQueue = [x for x in self.__class__.workqueue.getChildQueuesByRequest() if \
                    x['request_name'] == self.__class__.request_name]
            if request and 'Running' in [x['status'] for x in request]:
                self.assertTrue(childQueue,
                                "Running but can't get child queue")
                break
            if start + (60 * 20) < time.time():
                raise RuntimeError('timeout waiting for request to run')
            time.sleep(15)

    @attr("lifecycle")
    @recordException
    def test70WorkQueueFinished(self):
        """Request completed in workqueue"""
        start = time.time()
        while True:
            request = [x for x in self.__class__.workqueue.getJobStatusByRequest() if \
                    x['request_name'] == self.__class__.request_name]
            # request deleted from wq shortly after finishing, so may not appear here
            if not request or request == [
                    x for x in request
                    if x['status'] in ('Done', 'Failed', 'Canceled')
            ]:
                break
            if start + (60 * 20) < time.time():
                raise RuntimeError('timeout waiting for request to finish')
            time.sleep(15)

    @attr("lifecycle")
    @recordException
    def test80RequestFinished(self):
        """Request completed"""
        start = time.time()
        while True:
            self.__class__.request = self.__class__.reqmgr.getRequest(
                self.__class__.request_name)
            if self.__class__.request['RequestStatus'] in ('completed',
                                                           'failed',
                                                           'aborted'):
                break
            if start + (60 * 20) < time.time():
                raise RuntimeError('timeout waiting for request to finish')
            time.sleep(15)

    @attr("lifecycle")
    @recordException
    def test90RequestCloseOut(self):
        """Closeout request"""
        self.reqmgr.reportRequestStatus(self.__class__.request_name,
                                        "closed-out")
        self.__class__.request = self.__class__.reqmgr.getRequest(
            self.__class__.request_name)
        self.assertEqual('closed-out', self.__class__.request['RequestStatus'])
Пример #45
0
class RequestManagerTest(RESTBaseUnitTest):
    """
    Test RequestMgr Service client
    It will start RequestMgr RESTService
    Server DB is whatever env is set     
    
    This checks whether DS call makes without error and return the results.
    This test only test service call returns without error.
    The correctness of each function is tested in test/python/RequestManager_t/RequestMgr_t.py
    
    """
    def initialize(self):
        self.couchDBName = "reqmgr_t_0"
        self.config = RequestManagerConfig(
                'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel')
        dbUrl = os.environ.get("DATABASE", None)
        self.config.setDBUrl(dbUrl)        
        self.config.setFormatter('WMCore.WebTools.RESTFormatter')
        self.config.setupRequestConfig()
        self.config.setupCouchDatabase(dbName = self.couchDBName)
        self.config.setPort(8899)
        self.schemaModules = ["WMCore.RequestManager.RequestDB"]
                
        
    def setUp(self):
        RESTBaseUnitTest.setUp(self)
        self.testInit.setupCouch("%s" % self.couchDBName, "GroupUser", "ConfigCache", "ReqMgr")
        self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats")
        # logging stuff from TestInit is broken, setting myself
        l = logging.getLogger()
        l.setLevel(logging.DEBUG)
        self.params = {}
        self.params['endpoint'] = self.config.getServerUrl()
        self.reqService = RequestManagerDS(self.params)
        self.jsonSender = JSONRequests(self.config.getServerUrl())

        userName     = '******'
        groupName    = 'Li'
        teamName     = 'Tang'
        schema = utils.getAndSetupSchema(self,
                                         userName = userName,
                                         groupName = groupName,
                                         teamName = teamName)
        schema['ConfigCacheID'] = self.createConfig()
        schema['CouchDBName'] = self.couchDBName
        schema['CouchWorkloadDBName'] = self.couchDBName

        try:
            r = self.jsonSender.put('request', schema)
            try:
                self.requestName = r[0]['RequestName']
            except:
                self.requestName = r[0].values()[0]['RequestName']
        except Exception as ex:
            msg = traceback.format_exc()
            print("Exception during set up, reason: %s" % msg)
            raise ex

    def tearDown(self):
        self.config.deleteWorkloadCache()
        RESTBaseUnitTest.tearDown(self)
        self.testInit.tearDownCouch()

    def createConfig(self, bad = False):
        """
        _createConfig_

        Create a config of some sort that we can load out of ConfigCache
        """
        PSetTweak = {'process': {'outputModules_': ['ThisIsAName'],
                                 'ThisIsAName': {'dataset': {'dataTier': 'RECO',
                                                             'filterName': 'Filter'}}}}
        BadTweak  = {'process': {'outputModules_': ['ThisIsAName1', 'ThisIsAName2'],
                                 'ThisIsAName1': {'dataset': {'dataTier': 'RECO',
                                                             'filterName': 'Filter'}},
                                 'ThisIsAName2': {'dataset': {'dataTier': 'RECO',
                                                             'filterName': 'Filter'}}}}
        configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = self.couchDBName)
        configCache.createUserGroup(groupname = "testGroup", username = '******')
        if bad:
            configCache.setPSetTweaks(PSetTweak = BadTweak)
        else:
            configCache.setPSetTweaks(PSetTweak = PSetTweak)
        configCache.save()
        return configCache.getCouchID()

    @attr("integration")
    def testA_RequestManagerService(self):
        requestName = self.requestName
        
        request = self.reqService.getRequest(requestName)
        # minimal test : it's return type  and the some value inside
        self.assertEqual(type(request), dict)
        self.assertTrue(len(request) > 0)
        
        # Test putTeam
        self.reqService.putTeam("team_usa")
        self.assertTrue('team_usa' in self.jsonSender.get('team')[0])
        
        self.jsonSender.put('assignment/%s/%s' % ("team_usa", requestName))
        
        request = self.reqService.getAssignment(teamName = "team_usa")
        self.assertEqual(type(request), list)
        self.assertTrue(len(request) > 0)
       
        request = self.reqService.getAssignment(request = requestName)
        self.assertEqual(type(request), list)
        self.assertTrue(len(request) > 0)
        
        self.reqService.sendMessage(requestName,"error")
        self.reqService.putWorkQueue(requestName, "http://test_url")
        self.reqService.reportRequestProgress(requestName,
                        percent_complete = 100, percent_success = 90)
        
        self.reqService.updateRequestStatus(requestName, "running-open")
Пример #46
0
 def __init__(self, reqMgrUrl):
     self.reqMgrUrl = reqMgrUrl
     self.restSender = JSONRequests(reqMgrUrl)
     d = dict(endpoint=self.reqMgrUrl)
     self.reqMgrService = RequestManager(d)
Пример #47
0
 def __init__(self, reqMgrUrl):
     self.reqMgrUrl = reqMgrUrl
     self.restSender = JSONRequests(reqMgrUrl)
     d = dict(endpoint = self.reqMgrUrl)
     self.reqMgrService = RequestManager(d)
Пример #48
0
class RequestManagerTest(RESTBaseUnitTest):
    """
    Test RequestMgr Service client
    It will start RequestMgr RESTService
    Server DB is whatever env is set     
    
    This checks whether DS call makes without error and return the results.
    This test only test service call returns without error.
    The correctness of each function is tested in test/python/RequestManager_t/RequestMgr_t.py
    """
    def initialize(self):
        self.couchDBName = "reqmgr_t_0"
        self.config = RequestManagerConfig(
                'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel')
        dbUrl = os.environ.get("DATABASE", None)
        self.config.setDBUrl(dbUrl)        
        self.config.setFormatter('WMCore.WebTools.RESTFormatter')
        self.config.setupRequestConfig()
        self.config.setupCouchDatabase(dbName = self.couchDBName)
        self.config.setPort(8888)
        self.schemaModules = ["WMCore.RequestManager.RequestDB"]
        return
        
    def setUp(self):
        """
        setUP global values
        """
        RESTBaseUnitTest.setUp(self)
        self.testInit.setupCouch("%s" % self.couchDBName,
                                 "GroupUser", "ConfigCache")
        self.params = {}
        self.params['endpoint'] = self.config.getServerUrl()
        self.reqService = RequestManagerDS(self.params)
        self.jsonSender = JSONRequests(self.config.getServerUrl())
        self.requestSchema = getRequestSchema()
        self.jsonSender.put('group/PeopleLikeMe')
        self.jsonSender.put('user/[email protected]')
        self.jsonSender.put('group/PeopleLikeMe/me')
        self.jsonSender.put('version/CMSSW_3_5_8')
        r = self.jsonSender.put('request/' + self.requestSchema['RequestName'], 
                                self.requestSchema)
        self.requestName = r[0]['RequestName']
    
    def tearDown(self):
        self.config.deleteWorkloadCache()
        RESTBaseUnitTest.tearDown(self)
        self.testInit.tearDownCouch()

    @attr("integration")
    def testA_RequestManagerService(self):
        requestName = self.requestName
        
        request = self.reqService.getRequest(requestName)
        # minimal test : it's return type  and the some value inside
        self.assertEqual(type(request), dict)
        self.assertTrue(len(request) > 0)
        
        # Test putTeam
        self.reqService.putTeam("team_usa")
        self.assertTrue('team_usa' in self.jsonSender.get('team')[0])
        
        self.jsonSender.put('assignment/%s/%s' % ("team_usa", requestName))
        
        request = self.reqService.getAssignment(teamName = "team_usa")
        self.assertEqual(type(request), list)
        self.assertTrue(len(request) > 0)
       
        request = self.reqService.getAssignment(request = requestName)
        self.assertEqual(type(request), list)
        self.assertTrue(len(request) > 0)
        
        self.reqService.sendMessage(requestName,"error")
        self.reqService.putWorkQueue(requestName, "http://test_url")
        self.reqService.reportRequestProgress(requestName)
        self.reqService.reportRequestProgress(requestName,
                        percent_complete = 100, percent_success = 90)
        
        self.reqService.reportRequestStatus(requestName, "running")
        return
Пример #49
0
    cfg = cfg.WMStats
    import cherrypy
    from WMCore.Services.RequestManager.RequestManager import RequestManager
    from WMCore.Services.WorkQueue.WorkQueue import WorkQueue
    from WMCore.CherryPyThread.PeriodicWorker import PeriodicWorker
    from WMCore.WMStats.DataCollectTask import DataCollectTask
    import logging

    cherrypy.log.error_log.setLevel(logging.DEBUG)
    cherrypy.log.access_log.setLevel(logging.DEBUG)
    cherrypy.config["server.socket_port"] = cfg.port
    #def sayHello(test):
    #    print "Hello"
    #PeriodicWorker(sayHello, 5)

    # get reqmgr url from config
    reqmgrSvc = RequestManager({'endpoint': cfg.reqmgrURL})
    wqSvc = WorkQueue(cfg.globalQueueURL)
    wmstatSvc = WMStatSevice(cfg.couchURL)

    reqmgrTask = DataCollectTask(reqmgrSvc.getRequest, reqmgrDataFormat,
                                 wmstatSvc.uploadData)
    #reqmgrTask = DataCollectTask(reqmgrSvc.getRequestNames,  lambda x: x, wmstatSvc.uploadData)

    #wqTask = DataCollectTask(wqSvc.getTopLevelJobsByRequest, wqDataFormat, wmstatSvc.uploadData)

    reqmgrWorker = PeriodicWorker(reqmgrTask, cfg.pollInterval)
    #wqWorker = PeriodicWorker(wqTask, 200)

    cherrypy.quickstart()
Пример #50
0
class ReqMgrTest(RESTBaseUnitTest):
    """
    Basic test for the ReqMgr services.
    Setup is done off-screen in RESTBaseUnitTest - this makes
    things confusing
    
    """

    def setUp(self):
        """
        setUP global values
        Database setUp is done in base class
        """
        self.couchDBName = "reqmgr_t_0"
        RESTBaseUnitTest.setUp(self)
        self.testInit.setupCouch("%s" % self.couchDBName, "ConfigCache", "ReqMgr")
        self.testInit.setupCouch("%s_wmstats" % self.couchDBName,
                                 "WMStats")
        self.testInit.setupCouch("%s_acdc" % self.couchDBName,
                                 "ACDC", "GroupUser")
        reqMgrHost = self.config.getServerUrl()
        self.jsonSender = JSONRequests(reqMgrHost)

        self.params = {}
        self.params['endpoint'] = reqMgrHost
        self.reqService = RequestManager(self.params)

    def initialize(self):
        self.config = RequestManagerConfig(
                'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel')
        self.config.setFormatter('WMCore.WebTools.RESTFormatter')
        self.config.setupRequestConfig()
        self.config.setupCouchDatabase(dbName = self.couchDBName)
        self.config.setPort(12888)
        self.schemaModules = ["WMCore.RequestManager.RequestDB"]


    def tearDown(self):
        """
        tearDown 

        Tear down everything
        
        """
        RESTBaseUnitTest.tearDown(self)
        self.testInit.tearDownCouch()


    def createConfig(self, bad = False):
        """
        _createConfig_

        Create a config of some sort that we can load out of ConfigCache
        """
        
        PSetTweak = {'process': {'outputModules_': ['ThisIsAName'],
                                 'ThisIsAName': {'dataset': {'dataTier': 'RECO',
                                                             'filterName': 'Filter'}}}}

        BadTweak  = {'process': {'outputModules_': ['ThisIsAName1', 'ThisIsAName2'],
                                 'ThisIsAName1': {'dataset': {'dataTier': 'RECO',
                                                             'filterName': 'Filter'}},
                                 'ThisIsAName2': {'dataset': {'dataTier': 'RECO',
                                                             'filterName': 'Filter'}}}}

        configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = self.couchDBName)
        configCache.createUserGroup(groupname = "testGroup", username = '******')
        if bad:
            configCache.setPSetTweaks(PSetTweak = BadTweak)
        else:
            configCache.setPSetTweaks(PSetTweak = PSetTweak)
        configCache.save()

        return configCache.getCouchID()
    

    @attr("integration")
    def testA_testBasicSetUp(self):
        """
        _testBasicSetUp_
        
        Moving the tests that were in the setUp category out of it,
        mostly because I want to make sure that they don't fail inside
        the setUp statement.
        
        """
        if 'me' in self.jsonSender.get('user')[0]:
            self.jsonSender.delete('user/me')    
        self.assertFalse('me' in self.jsonSender.get('user')[0])
        self.assertEqual(self.jsonSender.put('user/[email protected]')[1], 200)
        self.assertTrue('me' in self.jsonSender.get('user')[0])

        if 'PeopleLikeMe' in self.jsonSender.get('group')[0]:
            self.jsonSender.delete('group/PeopleLikeMe')
        self.assertFalse('PeopleLikeMe' in self.jsonSender.get('group')[0])
        self.assertEqual(self.jsonSender.put('group/PeopleLikeMe')[1], 200)
        self.assertTrue( 'PeopleLikeMe' in self.jsonSender.get('group')[0])

        self.jsonSender.put('group/PeopleLikeMe/me')
        users = self.jsonSender.get('group/PeopleLikeMe')[0]['users']
        self.assertTrue('me' in users)
        groups = self.jsonSender.get('user/me')[0]['groups']
        self.assertTrue('PeopleLikeMe' in groups)
        groups2 = self.jsonSender.get('group?user=me')[0]
        self.assertTrue('PeopleLikeMe' in groups2)

        if 'White Sox' in self.jsonSender.get('team')[0]:
            self.jsonSender.delete(urllib.quote('team/White Sox'))
        self.assertFalse('White Sox' in self.jsonSender.get('team')[0])
        self.assertEqual(self.jsonSender.put(urllib.quote('team/White Sox'))[1], 200)
        self.assertTrue('White Sox' in self.jsonSender.get('team')[0])

        # some foreign key stuff to deal with
        schema = utils.getSchema()
        version = "version/" + schema["CMSSWVersion"]
        self.assertTrue(self.jsonSender.put(version)[1] == 200)
        self.assertTrue(schema["CMSSWVersion"] in self.jsonSender.get('version')[0])
        

    @attr("integration")
    def testB_ReReco(self):
        """
        _ReReco_

        Try a basic ReReco workflow
        
        """
        schema = utils.getAndSetupSchema(self)
        schema['RequestNumEvents'] = 100
        schema['SizePerEvent'] = 101
        configID = self.createConfig()
        schema["ConfigCacheID"] = configID
        schema["CouchDBName"] = self.couchDBName
        schema["CouchURL"]    = os.environ.get("COUCHURL")

        self.doRequest(schema)


    def doRequest(self, schema):
        """
        _doRequest_

        Run all tests on a basic ReReco workflow
        
        """
        requestName = schema['RequestName']
        self.assertRaises(HTTPException, self.jsonSender.delete, 'request/%s' % requestName)
        result = self.jsonSender.put('request/%s' % (requestName), schema)
        self.assertEqual(result[1], 200)
        requestName = result[0]['RequestName']

        self.assertEqual(self.jsonSender.get('request/%s' % requestName)[0]['RequestName'], requestName)

        self.jsonSender.put('request/%s?status=assignment-approved' % requestName)
        me = self.jsonSender.get('user/me')[0]
        self.assertTrue(requestName in me['requests'])
        self.assertEqual(self.jsonSender.put('request/%s?priority=5' % requestName)[1], 200)

        request = self.jsonSender.get('request/%s' % requestName)[0]
        self.assertEqual(request['RequestPriority'], 5)

        # Check LFN Bases
        self.assertEqual(request['UnmergedLFNBase'], '/store/unmerged')
        self.assertEqual(request['MergedLFNBase'], '/store/data')

        # Check Num events
        self.assertEqual(request['RequestNumEvents'], 100)
        self.assertEqual(request['SizePerEvent'], 101)

        # only certain transitions allowed
        #self.assertEqual(self.jsonSender.put('request/%s?status=running' % requestName)[1], 400)
        self.assertRaises(HTTPException, self.jsonSender.put,'request/%s?status=running' % requestName)
        request = self.jsonSender.get('request/%s' % requestName)[0]
        self.assertEqual(request['RequestStatus'], 'assignment-approved')

        self.assertTrue(self.jsonSender.put(urllib.quote('assignment/White Sox/%s' % requestName))[1] == 200)
        requestsAndSpecs = self.jsonSender.get(urllib.quote('assignment/White Sox'))[0]
        self.assertTrue(requestName in requestsAndSpecs[0])
        workloadHelper = WMWorkloadHelper()
        workloadHelper.load(requestsAndSpecs[0][1]) 
        self.assertEqual(workloadHelper.getOwner()['Requestor'], "me")
        self.assertEqual(self.jsonSender.get('assignment?request=%s'% requestName)[0], ['White Sox'])
        self.assertEqual(self.jsonSender.get('request/%s' % requestName)[0]['teams'], ['White Sox'])

        agentUrl = 'http://cmssrv96.fnal.gov/workqueue'
        self.jsonSender.put('workQueue/%s?url=%s'% (requestName, urllib.quote(agentUrl)) )
        self.assertEqual(self.jsonSender.get('workQueue/%s' % requestName)[0][0], agentUrl)
        request = self.jsonSender.get('request/%s' % requestName)[0]
        self.assertEqual(request['RequestStatus'], 'acquired')

        self.jsonSender.post('request/%s?events_written=10&files_merged=1' % requestName)
        self.jsonSender.post('request/%s?events_written=20&files_merged=2&percent_success=99.9' % requestName)
        request = self.jsonSender.get('request/%s' % requestName)[0]
        self.assertEqual(len(request['RequestUpdates']), 2)
        self.assertEqual(request['RequestUpdates'][0]['files_merged'], 1)
        self.assertEqual(request['RequestUpdates'][1]['events_written'], 20)
        self.assertEqual(request['RequestUpdates'][1]['percent_success'], 99.9)

        message = "The sheriff is near"
        jsonMessage = json.dumps(message)
        self.jsonSender.put('message/%s' % requestName, message)
        messages = self.jsonSender.get('message/%s' % requestName)
        #self.assertEqual(messages[0][0][0], message)
        for status in ['running-open', 'running-closed', 'completed']:
            self.jsonSender.put('request/%s?status=%s' % (requestName, status))

        # campaign
        self.jsonSender.put('campaign/%s' % 'TestCampaign')
        campaigns = self.jsonSender.get('campaign')[0]
        self.assertTrue('TestCampaign' in campaigns.keys())
        self.jsonSender.put('campaign/%s/%s' % ('TestCampaign', requestName))
        requestsInCampaign = self.jsonSender.get('campaign/%s' % 'TestCampaign')[0]
        self.assertTrue(requestName in requestsInCampaign.keys())

        req = self.jsonSender.get('request/%s' % requestName)[0]
        self.assertEqual(req['Campaign'], 'TestCampaign')
        self.jsonSender.delete('request/%s' % requestName)
        

    @attr("integration")
    def testC_404Errors(self):
        """
        _404Errors_

        Do some things that generate 404 errors.  This
        should be limited to requests for objects that do not
        exist.
        """
        badName = 'ThereIsNoWayThisNameShouldExist'

        # First, try to find a non-existent request
        # This should throw a 404 error.
        # The request name should not be in it
        self.checkForError(cls = 'request', badName = badName, exitCode = 404,
                           message = 'Given requestName not found')

        # Now look for non-existent user
        self.checkForError(cls = 'user', badName = badName, exitCode = 404,
                           message = 'Cannot find user')

        # Now try non-existent campaign
        self.checkForError(cls = 'campaign', badName = badName, exitCode = 404,
                           message = "Cannot find campaign")

        # Now try invalid message
        # This raises a requestName error because it searches for the request
        self.checkForError(cls = 'message', badName = badName, exitCode = 404,
                           message = "Given requestName not found", testEmpty = False)

        # Check for assignments (no teams or requests)
        # This raises a team error because it tries to load teams out first
        self.checkForError(cls = 'assignment', badName = badName, exitCode = 404,
                           message = 'Cannot find team')


    @attr("integration")
    def testD_400Errors(self):
        """
        _400Errors_

        These are failures created by invalid input, such as sending
        args to a request when it doesn't accept any.  They should
        generatore 400 Errors
        
        """
        badName = 'ThereIsNoWayThisNameShouldExist'
        
        # Attempt to send arguments to a function that doesn't accept them.
        self.checkForError(cls = 'team', badName = badName, exitCode = 400,
                           message = "Invalid input: Arguments added where none allowed")

        # Recheck for versions
        self.checkForError(cls = 'version', badName = badName, exitCode = 400,
                           message = "Invalid input: Arguments added where none allowed")

        # Break the validation
        self.checkForError(cls = 'user', badName = '!', exitCode = 400,
                           message = 'Invalid input: Input data failed validation')
        

    def checkForError(self, cls, badName, exitCode, message, testEmpty = True):
        """
        _checkForError_

        Generic function for checking for errors in JSON commands

        Does a basic check on type cls searching for name badName which hopefull
        does not exist.

        Checks to make sure that it exits with code exitCode, and that
        the error contains the string message.

        Also checks to make sure that name badName is NOT in the output

        testEmpty for those that don't handle calls to the main (i.e., who require
        an argument)
        
        """
        raises = False
        # First assert that the test to be tested is empty
        if testEmpty:
            result = self.jsonSender.get(cls)
            self.assertTrue(type(result[0]) in [type([]), type({})])
        # Next, test
        try:
            result = self.jsonSender.get('%s/%s' % (cls, badName))
        except HTTPException as ex:
            raises = True
            self.assertEqual(ex.status, exitCode)
            self.assertTrue(message in ex.result)
            self.assertFalse(badName in ex.result)
        self.assertTrue(raises)


    @attr("integration")
    def testE_CheckStatusChanges(self):
        """
        _CheckStatusChanges_
        
        Check status changes for a single request.  See whether
        we can move the request through the proper chain.  Figure
        out what happens when we fail.
        
        """
        myThread = threading.currentThread()

        userName     = '******'
        groupName    = 'Li'
        teamName     = 'Tang'
        schema       = utils.getAndSetupSchema(self,
                                               userName = userName,
                                               groupName = groupName,
                                               teamName = teamName)
        configID = self.createConfig()
        schema["ConfigCacheID"] = configID
        schema["CouchDBName"] = self.couchDBName
        schema["CouchURL"]    = os.environ.get("COUCHURL")
        result = self.jsonSender.put('request/testRequest', schema)
        self.assertEqual(result[1], 200)
        requestName = result[0]['RequestName']

        # There should only be one request in the DB
        result = GetRequest.requestID(requestName = requestName)
        self.assertEqual(result, 1)
        result = self.jsonSender.get('request/%s' % requestName)
        self.assertEqual(result[0]['Group'], groupName)
        self.assertEqual(result[0]['Requestor'], userName)

        # Let's see what we can do in terms of setting status
        self.changeStatusAndCheck(requestName = requestName,
                                  statusName  = 'new')

        # Let's try an illegal status change, just for the hell of it
        raises = False
        try:
            self.jsonSender.put('request/%s?status=negotiating' % requestName)
        except HTTPException as ex:
            raises = True
            self.assertEqual(ex.status, 403)
            self.assertTrue('Failed to change status' in ex.result)
            self.assertFalse(requestName in ex.result)
        self.assertTrue(raises)

        # Now, let's try a totally bogus status
        raises = False
        try:
            self.jsonSender.put('request/%s?status=bogus' % requestName)
        except HTTPException as ex:
            raises = True
            self.assertEqual(ex.status, 403)
            self.assertTrue('Failed to change status' in ex.result)
            self.assertFalse(requestName in ex.result)
        self.assertTrue(raises)

        # We should still be in new
        result = self.jsonSender.get('request/%s' % requestName)
        self.assertEqual(result[0]['RequestStatus'], 'new')

        # Let's go on in a full loop
        self.changeStatusAndCheck(requestName = requestName,
                                  statusName  = 'testing-approved')
        self.changeStatusAndCheck(requestName = requestName,
                                  statusName  = 'testing')
        self.changeStatusAndCheck(requestName = requestName,
                                  statusName  = 'tested')
        self.changeStatusAndCheck(requestName = requestName,
                                  statusName  = 'assignment-approved')

        # This should fail, as you cannot assign a request without a team
        raises = False
        try:
            self.changeStatusAndCheck(requestName = requestName,
                                      statusName  = 'assigned')
        except HTTPException as ex:
            raises = True
            self.assertTrue('Cannot change status without a team' in ex.result)
        self.assertTrue(raises)

        self.jsonSender.put(urllib.quote('assignment/%s/%s' % (teamName, requestName)))
        self.changeStatusAndCheck(requestName = requestName,
                                  statusName  = 'negotiating')
        self.changeStatusAndCheck(requestName = requestName,
                                  statusName  = 'acquired')
        self.changeStatusAndCheck(requestName = requestName,
                                  statusName  = 'running-open')
        self.changeStatusAndCheck(requestName = requestName,
                                  statusName  = 'running-closed')
        self.changeStatusAndCheck(requestName = requestName,
                                  statusName  = 'completed')
        self.changeStatusAndCheck(requestName = requestName,
                                  statusName  = 'closed-out')


    def changeStatusAndCheck(self, requestName, statusName):
        """
        _changeStatusAndCheck_

        Change the status of a request and make sure that
        the request actually did it.
        
        """
        self.jsonSender.put('request/%s?status=%s' % (requestName, statusName))
        result = self.jsonSender.get('request/%s' % requestName)
        self.assertEqual(result[0]['RequestStatus'], statusName)
        

    def loadWorkload(self, requestName):
        """
        _loadWorkload_

        Load the workload from couch after we've saved it there.
        
        """
        workload = WMWorkloadHelper()
        url      = '%s/%s/%s/spec' % (os.environ['COUCHURL'], self.couchDBName,
                                      requestName)
        workload.load(url)
        return workload


    def testF_TestWhitelistBlacklist(self):
        """
        _TestWhitelistBlacklist_

        Test whether or not we can assign the block/run blacklist/whitelist
        
        """
        userName     = '******'
        groupName    = 'Li'
        teamName     = 'Tang'
        schema       = utils.getAndSetupSchema(self,
                                               userName = userName,
                                               groupName = groupName,
                                               teamName = teamName)
        schema['RunWhitelist'] = [1, 2, 3]
        schema['RunBlacklist'] = [4, 5, 6]
        schema['BlockWhitelist'] = ['/dataset/dataset/dataset#alpha']
        schema['BlockBlacklist'] = ['/dataset/dataset/dataset#beta']
        configID = self.createConfig()
        schema["ConfigCacheID"] = configID
        schema["CouchDBName"] = self.couchDBName
        schema["CouchURL"]    = os.environ.get("COUCHURL")

        result = self.jsonSender.put('request/testRequest', schema)
        self.assertEqual(result[1], 200)
        requestName = result[0]['RequestName']

        workload = self.loadWorkload(requestName = requestName)
        self.assertEqual(workload.data.tasks.DataProcessing.input.dataset.runs.whitelist, schema['RunWhitelist'])
        self.assertEqual(workload.data.tasks.DataProcessing.input.dataset.runs.blacklist, schema['RunBlacklist'])
        self.assertEqual(workload.data.tasks.DataProcessing.input.dataset.blocks.whitelist, schema['BlockWhitelist'])
        self.assertEqual(workload.data.tasks.DataProcessing.input.dataset.blocks.blacklist, schema['BlockBlacklist'])

        req = self.jsonSender.get('request/%s' % requestName)
        self.assertTrue('Site Blacklist' in req[0])
        self.assertTrue('Site Whitelist' in req[0])

        schema['BlockBlacklist'] = {'1': '/dataset/dataset/dataset#beta'}

        try:
            raises = False
            result = self.jsonSender.put('request/testRequest', schema)
        except HTTPException as ex:
            raises = True
            self.assertEqual(ex.status, 400)
            print ex.result
            self.assertTrue("Error in Workload Validation: Argument BlockBlacklist type is incorrect in schema." in ex.result)
            pass
        self.assertTrue(raises)

        schema['BlockBlacklist'] = ['/dataset/dataset/dataset#beta']
        schema['RunWhitelist']   = {'1': '/dataset/dataset/dataset#beta'}

        try:
            raises = False
            result = self.jsonSender.put('request/testRequest', schema)
        except HTTPException as ex:
            raises = True
            self.assertEqual(ex.status, 400)
            self.assertTrue("Error in Workload Validation: Argument RunWhitelist type is incorrect in schema." in ex.result)
            pass
        self.assertTrue(raises)

        schema['RunWhitelist'] = ['hello', 'how', 'are', 'you']
        try:
            raises = True
            result = self.jsonSender.put('request/testRequest', schema)
        except HTTPException as ex:
            raises = True
            self.assertEqual(ex.status, 400)
            self.assertTrue("Error in Workload Validation: Argument RunWhitelist doesn't pass validation." in ex.result)
            pass
        self.assertTrue(raises)


    def testG_AddDuplicateUser(self):
        """
        _AddDuplicateUser_

        Test and see if we get a sensible error when adding a duplicate user.
        
        """
        userName     = '******'
        groupName    = 'Li'
        teamName     = 'Tang'
        schema       = utils.getAndSetupSchema(self,
                                               userName = userName,
                                               groupName = groupName,
                                               teamName = teamName)
        raises = False
        try:
            self.jsonSender.put('group/%s/%s' % (groupName, userName))
        except HTTPException as ex:
            self.assertTrue("User/Group Already Linked in DB" in ex.result)
            self.assertEqual(ex.status, 400)
            raises = True
        self.assertTrue(raises)


    def testH_RemoveSoftwareVersion(self):
        """
        _RemoveSoftwareVersion_

        Remove the software version after submitting the request.  See what that does.
        
        """
        myThread = threading.currentThread()
        userName     = '******'
        groupName    = 'Li'
        teamName     = 'Tang'
        schema       = utils.getAndSetupSchema(self,
                                               userName = userName,
                                               groupName = groupName,
                                               teamName = teamName)
        configID = self.createConfig()
        schema["ConfigCacheID"] = configID
        schema["CouchDBName"] = self.couchDBName
        schema["CouchURL"]    = os.environ.get("COUCHURL")
        result = self.jsonSender.put('request/testRequest', schema)
        self.assertEqual(result[1], 200)
        requestName = result[0]['RequestName']

        req = self.jsonSender.get('request/%s' % requestName)[0]
        self.assertEqual(req['SoftwareVersions'], [schema["CMSSWVersion"]])

        # Delete software versions and make sure they're gone from the DB
        SoftwareManagement.removeSoftware(softwareName = schema["CMSSWVersion"],
                                          scramArch = schema["ScramArch"])
        versions = myThread.dbi.processData("SELECT * FROM reqmgr_software")[0].fetchall()
        self.assertEqual(versions, [])
        assocs = myThread.dbi.processData("SELECT * FROM reqmgr_software_dependency")[0].fetchall()
        self.assertEqual(assocs, [])
        
        req = self.jsonSender.get('request/%s' % requestName)[0]
        self.assertEqual(req['SoftwareVersions'], [schema["CMSSWVersion"]])
        

    def testI_CheckConfigIDs(self):
        """
        _CheckConfigIDs_

        Check to see if we can pull out the ConfigIDs by request
        
        """
        userName     = '******'
        groupName    = 'Li'
        teamName     = 'Tang'
        schema       = utils.getAndSetupSchema(self,
                                               userName = userName,
                                               groupName = groupName,
                                               teamName = teamName)
        # Set some versions
        schema['ProcessingVersion'] = '2012'
        schema['AcquisitionEra']    = 'ae2012'
        schema["PrimaryDataset"]    = "ReallyFake"
        schema["RequestNumEvents"]  = 100

        configID = self.createConfig()
        schema["CouchDBName"] = self.couchDBName
        schema["CouchURL"]    = os.environ.get("COUCHURL")
        schema["ConfigCacheID"] = configID
        schema["InputDataset"]     = '/MinimumBias/Run2010B-RelValRawSkim-v1/RAW'

        result = self.jsonSender.put('request/testRequest', schema)
        self.assertEqual(result[1], 200)
        requestName = result[0]['RequestName']

        result = self.jsonSender.get('configIDs?prim=MinimumBias&proc=Run2010B-RelValRawSkim-v1&tier=RAW')[0]
        print result
        self.assertTrue(requestName in result.keys())
        self.assertTrue(configID in result[requestName][0])
        
        
    def testJ_CheckRequestCloning(self):
        myThread = threading.currentThread()
        userName     = '******'
        groupName    = 'Li'
        teamName     = 'Tang'
        schema       = utils.getAndSetupSchema(self,
                                               userName = userName,
                                               groupName = groupName,
                                               teamName = teamName)
        configID = self.createConfig()
        schema["ConfigCacheID"] = configID
        schema["CouchDBName"] = self.couchDBName
        schema["CouchURL"]    = os.environ.get("COUCHURL")
        schema["AcquisitionEra"] = "NewEra"
        result = self.jsonSender.put("request", schema)
        self.assertEqual(result[1], 200)
        requestName = result[0]["RequestName"]
        acquisitionEra = result[0]["AcquisitionEra"]
        self.assertTrue(schema["AcquisitionEra"], acquisitionEra)
        # set some non-default priority
        # when cloning a request which had some non default priority,
        # the priority values were lost when creating a cloned request, the
        # default values were lost. Change it here to specifically catch this case.
        priority = 300
        result = self.jsonSender.put("request/%s?priority=%s" % (requestName, priority))        
        self.assertEqual(result[1], 200)
        # get the original request from the server, although the variable result
        # shall have the same stuff in
        response = self.jsonSender.get("request/%s" % requestName)
        origRequest = response[0]
        self.assertEquals(origRequest["AcquisitionEra"], acquisitionEra)
        # test that the priority was correctly set in the brand-new request
        self.assertEquals(origRequest["RequestPriority"], priority)
        
        # test cloning not existing request
        self.assertRaises(HTTPException, self.jsonSender.put, "clone/%s" % "NotExistingRequestName")
        # correct attempt to clone the request
        # this is the new request, it'll have different name
        result = self.jsonSender.put("clone/%s" % requestName)
        # get the cloned request from the server
        respose = self.jsonSender.get("request/%s" % result[0]["RequestName"])
        clonedRequest = respose[0]
        # these request arguments shall differ in the cloned request:
        toDiffer = ["RequestName", "RequestStatus"]
        for differ in toDiffer:
            self.assertNotEqual(origRequest[differ], clonedRequest[differ])
        # check the desired status of the cloned request
        self.assertEquals(clonedRequest["RequestStatus"], "assignment-approved",
                          "Cloned request status should be 'assignment-approved', not '%s'." %
                          clonedRequest["RequestStatus"])
        # don't care about these two (they will likely be the same in the unittest
        # since the brand new request injection as well as the cloning probably
        # happen at roughly the same time)
        toDiffer.extend(["RequestDate", "timeStamp", "RequestWorkflow"])
        for differ in toDiffer:
            del origRequest[differ]
            del clonedRequest[differ]
        # check the request dictionaries
        self.assertEquals(len(origRequest), len(clonedRequest))
        for k1, k2 in zip(sorted(origRequest.keys()), sorted(clonedRequest.keys())):
            msg = ("Request values: original: %s: %s cloned: %s: %s differ" %
                   (k1, origRequest[k1], k2, clonedRequest[k2]))
            self.assertEqual(origRequest[k1], clonedRequest[k2], msg)
            

    def testK_CheckRequestFailsInjectionForbiddenInputArg(self):
        myThread = threading.currentThread()
        userName     = '******'
        groupName    = 'Li'
        teamName     = 'Tang'
        schema       = utils.getAndSetupSchema(self,
                                               userName = userName,
                                               groupName = groupName,
                                               teamName = teamName)
        from WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel import deprecatedRequestArgs
        for deprec in deprecatedRequestArgs:
            schema = utils.getSchema(groupName=groupName, userName=userName)
            schema[deprec] = "something"
            self.assertRaises(HTTPException, self.jsonSender.put, "request", schema)

    def setupACDCDatabase(self, collectionName, taskPath,
                          user, group):
        """
        _setupACDCDatabase_

        Populate an ACDC database with bogus records
        associated to certain collection name, user and task path.
        """
        acdcServer = CouchService(url = self.testInit.couchUrl,
                                  database = "%s_acdc" % self.couchDBName)
        owner = acdcServer.newOwner(group, user)
        testCollection = CouchCollection(database = self.testInit.couchDbName,
                                          url = self.testInit.couchUrl,
                                          name = collectionName)
        testCollection.setOwner(owner)
        testFileset = CouchFileset(database = self.testInit.couchDbName,
                                    url = self.testInit.couchUrl,
                                    name = taskPath)
        testCollection.addFileset(testFileset)

        testFiles = []
        for _ in range(5):
            testFile = File(lfn = makeUUID(), size = random.randint(1024, 4096),
                            events = random.randint(1024, 4096))
            testFiles.append(testFile)

        testFileset.add(testFiles)

    def testL_CascadeCloseOutAnnnouncement(self):
        """
        _testL_CascadeCloseOutAnnouncement_

        Test the cascade closeout REST call, also
        check that when announced a request deletes all ACDC records in the system.
        """
        userName     = '******'
        groupName    = 'Li'
        teamName     = 'Tang'
        schema       = utils.getAndSetupSchema(self,
                                               userName = userName,
                                               groupName = groupName,
                                               teamName = teamName)
        configID = self.createConfig()
        schema["ConfigCacheID"] = configID
        schema["CouchDBName"] = self.couchDBName
        schema["CouchURL"]    = os.environ.get("COUCHURL")

        result = self.jsonSender.put("request", schema)[0]
        originalRequest = result['RequestName']
        self.setupACDCDatabase(originalRequest, "/%s/DataProcessing" % originalRequest,
                               result['Requestor'], result['Group'])
        depth = 2
        nReq = 3
        requests = [originalRequest]
        def createChildrenRequest(parentRequest, i, nReq):
            createdRequests = []
            resubSchema = utils.getResubmissionSchema(parentRequest, "/%s/DataProcessing" % parentRequest,
                                                      groupName, userName)
            result = self.jsonSender.put("request", resubSchema)[0]
            requestName = result['RequestName']
            self.setupACDCDatabase(requestName, "/%s/DataProcessing" % requestName, result['Requestor'], result['Group'])
            createdRequests.append(requestName)
            if i:
                for _ in range(nReq):
                    createdRequests.extend(createChildrenRequest(requestName, i - 1, nReq))
            return createdRequests
        requests.extend(createChildrenRequest(originalRequest, depth, nReq))
        for request in requests:
            self.changeStatusAndCheck(request, 'assignment-approved')
        for request in requests:
            self.jsonSender.put("assignment?team=%s&requestName=%s" % (teamName, request))
        for status in ['acquired',
                       'running-open', 'running-closed',
                       'completed']:
            for request in requests:
                self.changeStatusAndCheck(request, status)
        self.jsonSender.post('closeout?requestName=%s&cascade=True' % originalRequest)
        svc = CouchService(url = self.testInit.couchUrl,
                                  database = "%s_acdc" % self.couchDBName)

        owner = svc.newOwner(groupName, userName)
        for request in requests:
            result = self.jsonSender.get('request/%s' % request)
            self.assertEqual(result[0]['RequestStatus'], 'closed-out')
            testCollection = CouchCollection(database = self.testInit.couchDbName,
                                             url = self.testInit.couchUrl,
                                             name = request)
            testCollection.setOwner(owner)
            testCollection.populate()
            self.assertNotEqual(len(testCollection["filesets"]), 0)

        self.jsonSender.post('announce?requestName=%s&cascade=True' % originalRequest)
        for request in requests:

            result = self.jsonSender.get('request/%s' % request)
            self.assertEqual(result[0]['RequestStatus'], 'announced')
            testCollection = CouchCollection(database = self.testInit.couchDbName,
                                             url = self.testInit.couchUrl,
                                             name = request)
            testCollection.setOwner(owner)
            testCollection.populate()
            self.assertEqual(len(testCollection["filesets"]), 0)

            
    def testM_PutRequestStats(self):
        userName     = '******'
        groupName    = 'Bryant'
        teamName     = 'Lakers'
        schema       = utils.getAndSetupSchema(self,
                                               userName = userName,
                                               groupName = groupName,
                                               teamName = teamName)
        configID = self.createConfig()
        schema["ConfigCacheID"] = configID
        schema["CouchDBName"] = self.couchDBName
        schema["CouchURL"]    = os.environ.get("COUCHURL")

        result = self.jsonSender.put("request", schema)[0]
        originalRequest = result['RequestName']
        stats = {'total_jobs': 100, 'input_events': 100, 'input_lumis': 100, 'input_num_files': 100}
        result = self.reqService.putRequestStats(originalRequest, stats)
        self.assertEqual(result['RequestName'], originalRequest)