Example #1
0
def getSpec(request, reqSpecs=None):
    "Get request from workload cache"
    if reqSpecs and request['RequestName'] in reqSpecs:
        return reqSpecs[request['RequestName']]
    url = str('%s/%s/spec' % (reqmgrCacheUrl(), request['RequestName']))
    mgr = RequestHandler()
    data = mgr.getdata(url, params={}, cert=cert(), ckey=ckey())
    return pickle.loads(data)
Example #2
0
def findParent(dataset):
    "Helper function to find a parent of the dataset"
    url = '%s/datasetparents' % dbsUrl()
    params = {'dataset': dataset}
    headers = {'Accept': 'application/json'}
    mgr = RequestHandler()
    data = mgr.getdata(url, params=params, headers=headers, cert=cert(), ckey=ckey())
    return [str(i['parent_dataset']) for i in json.loads(data)]
Example #3
0
def getWorkflows(state):
    "Get list of workflows from ReqMgr2 data-service"
    url = '%s/data/request' % reqmgrUrl()
    headers = {'Accept': 'application/json'}
    params = {'status': state}
    mgr = RequestHandler()
    res = mgr.getdata(url, params=params, headers=headers, ckey=ckey(), cert=cert())
    data = json.loads(res)
    return data.get('result', [])
Example #4
0
def getNodes(kind):
    "Get list of PhEDEx nodes"
    params = {}
    headers = {'Accept': 'application/json'}
    url = '%s/nodes' % phedexUrl()
    mgr = RequestHandler()
    data = mgr.getdata(url, params=params, headers=headers, ckey=ckey(), cert=cert())
    nodes = json.loads(data)['phedex']['node']
    return [node['name'] for node in nodes if node['kind'] == kind]
Example #5
0
def getWorkflow(requestName):
    "Get list of workflow info from ReqMgr2 data-service for given request name"
    headers = {'Accept': 'application/json'}
    params = {}
    url = '%s/data/request/%s' % (reqmgrUrl(), requestName)
    mgr = RequestHandler()
    res = mgr.getdata(url, params=params, headers=headers, ckey=ckey(), cert=cert())
    data = json.loads(res)
    return data.get('result', [])
Example #6
0
def getRequest(url, params):
    "Helper function to GET data from given URL"
    mgr = RequestHandler()
    headers = {'Accept': 'application/json'}
    verbose = 0
    if 'verbose' in params:
        verbose = params['verbose']
        del params['verbose']
    data = mgr.getdata(url, params, headers, ckey=ckey(), cert=cert(), verbose=verbose)
    return data
Example #7
0
def getNodesForId(phedexid):
    "Helper function to get nodes for given phedex id"
    url = '%s/requestlist' % phedexUrl()
    params = {'request': str(phedexid)}
    headers = {'Accept': 'application/json'}
    mgr = RequestHandler()
    data = mgr.getdata(url, params, headers, ckey=ckey(), cert=cert())
    items = json.loads(data)['phedex']['request']
    nodes = [n['name'] for i in items for n in i['node']]
    return list(set(nodes))
Example #8
0
def getNodeQueues():
    "Helper function to fetch nodes usage from PhEDEx data service"
    headers = {'Accept': 'application/json'}
    params = {}
    mgr = RequestHandler()
    url = '%s/nodeusagehistory' % phedexUrl()
    res = mgr.getdata(url, params=params, headers=headers, ckey=ckey(), cert=cert())
    data = json.loads(res)
    ret = defaultdict(int)
    for node in data['phedex']['node']:
        for usage in node['usage']:
            ret[node['name']] += int(usage['miss_bytes'] / 1023.**4) #in TB
    return ret
Example #9
0
def getNodes(kind):
    "Get list of PhEDEx nodes"
    params = {}
    headers = {'Accept': 'application/json'}
    url = 'https://cmsweb.cern.ch/phedex/datasvc/json/prod/nodes'
    mgr = RequestHandler()
    data = mgr.getdata(url,
                       params=params,
                       headers=headers,
                       ckey=ckey(),
                       cert=cert())
    nodes = json.loads(data)['phedex']['node']
    return [node['name'] for node in nodes if node['kind'] == kind]
Example #10
0
def getWorkflow(requestName):
    "Get list of workflow info from ReqMgr2 data-service for given request name"
    headers = {'Accept': 'application/json'}
    params = {}
    url = '%s/data/request/%s' % (reqmgrUrl(), requestName)
    mgr = RequestHandler()
    res = mgr.getdata(url,
                      params=params,
                      headers=headers,
                      ckey=ckey(),
                      cert=cert())
    data = json.loads(res)
    return data.get('result', [])
Example #11
0
def getWorkflows(state):
    "Get list of workflows from ReqMgr2 data-service"
    url = '%s/data/request' % reqmgrUrl()
    headers = {'Accept': 'application/json'}
    params = {'status': state}
    mgr = RequestHandler()
    res = mgr.getdata(url,
                      params=params,
                      headers=headers,
                      ckey=ckey(),
                      cert=cert())
    data = json.loads(res)
    return data.get('result', [])
Example #12
0
def getNodeQueues():
    "Helper function to fetch nodes usage from PhEDEx data service"
    headers = {'Accept': 'application/json'}
    params = {}
    mgr = RequestHandler()
    url = '%s/nodeusagehistory' % phedexUrl()
    res = mgr.getdata(url, params=params, headers=headers, ckey=ckey(), cert=cert())
    data = json.loads(res)
    ret = defaultdict(int)
    for node in data['phedex']['node']:
        for usage in node['usage']:
            ret[node['name']] += int(usage['miss_bytes'] / 1023.**4) #in TB
    return ret
Example #13
0
def workqueueRequests(state=None):
    "Helper functions to get requests from WorkQueue"
    url = workqueueView('jobsByRequest')
    if state:
        pass # we may need to use state when we'll query WorkQueue
    params = {}
    headers = {'Accept': 'application/json'}
    mgr = RequestHandler()
    data = mgr.getdata(url, params=params, headers=headers, cert=cert(), ckey=ckey())
    data = json.loads(data)
    rdict = {}
    for row in data.get('rows', []):
        rdict[row['key']] = row['value']
    return rdict
Example #14
0
def getRequest(url, params):
    "Helper function to GET data from given URL"
    mgr = RequestHandler()
    headers = {'Accept': 'application/json'}
    verbose = 0
    if 'verbose' in params:
        verbose = params['verbose']
        del params['verbose']
    data = mgr.getdata(url,
                       params,
                       headers,
                       ckey=ckey(),
                       cert=cert(),
                       verbose=verbose)
    return data
Example #15
0
 def setUp(self):
     "Setup MicroService for testing"
     self.app = ServiceManager()
     config = TestConfig
     manager = 'WMCore_t.Services_t.MicroService_t.MicroService_t.ServiceManager'
     config.views.data.manager = manager
     config.manager = manager
     mount = '/microservice'
     self.mgr = RequestHandler()
     self.port = config.main.port
     self.url = 'http://localhost:%s%s/data' % (self.port, mount)
     cherrypy.config["server.socket_port"] = self.port
     self.server = RestInterface(self.app, config, mount)
     cherrypy.tree.mount(self.server, mount)
     cherrypy.engine.start()
Example #16
0
 def getUrlOpener(self):
     """
     method getting an HTTPConnection, it is used by the constructor such
     that a sub class can override it to have different type of connection
     i.e. - if it needs authentication, or some fancy handler
     """
     return RequestHandler(config={'timeout': 300, 'connecttimeout': 300})
Example #17
0
class MicroServiceTest(unittest.TestCase):
    "Unit test for MicroService module"

    def setUp(self):
        "Setup MicroService for testing"
        self.app = ServiceManager()
        config = TestConfig
        manager = 'WMCore_t.Services_t.MicroService_t.MicroService_t.ServiceManager'
        config.views.data.manager = manager
        config.manager = manager
        mount = '/microservice'
        self.mgr = RequestHandler()
        self.port = config.main.port
        self.url = 'http://localhost:%s%s/data' % (self.port, mount)
        cherrypy.config["server.socket_port"] = self.port
        self.server = RestInterface(self.app, config, mount)
        cherrypy.tree.mount(self.server, mount)
        cherrypy.engine.start()

    def tearDown(self):
        "Tear down MicroService"
        cherrypy.engine.exit()
        cherrypy.engine.stop()

    def postRequest(self, params):
        "Perform POST request to our MicroService"
        headers = {'Content-type': 'application/json'}
        print("### post call %s params=%s headers=%s" %
              (self.url, params, headers))
        data = self.mgr.getdata(self.url, params=params, headers=headers, \
                verb='POST', cert=cert(), ckey=ckey())
        print("### post call data %s" % data)
        return json.loads(data)

    def test_getState(self):
        "Test function for getting state of the MicroService"
        url = '%s/status' % self.url
        data = self.mgr.getdata(url, params={})
        state = "bla"
        data = {"request": {"state": state}}
        self.postRequest(data)
        data = self.mgr.getdata(url, params={})
        data = json.loads(data)
        print("### url=%s, data=%s" % (url, data))
        for row in data['result']:
            if 'state' in row:
                self.assertEqual(state, row['state'])
Example #18
0
    def __init__(self, url='http://localhost', idict=None):
        """
        url should really be host - TODO fix that when have sufficient code
        coverage and change _getURLOpener if needed
        """
        if not idict:
            idict = {}
        dict.__init__(self, idict)
        self.pycurl = idict.get('pycurl', None)
        self.capath = idict.get('capath', None)
        if self.pycurl:
            self.reqmgr = RequestHandler()

        # set up defaults
        self.setdefault("accept_type", 'text/html')
        self.setdefault("content_type", 'application/x-www-form-urlencoded')
        self.additionalHeaders = {}

        # check for basic auth early, as if found this changes the url
        urlComponent = sanitizeURL(url)
        if urlComponent['username'] is not None:
            self.addBasicAuth(urlComponent['username'],
                              urlComponent['password'])
            url = urlComponent['url']  # remove user, password from url

        self.setdefault("host", url)

        # then update with the incoming dict
        self.update(idict)

        self['endpoint_components'] = urlparse.urlparse(self['host'])

        # If cachepath = None disable caching
        if 'cachepath' in idict and idict['cachepath'] is None:
            self["req_cache_path"] = None
        else:
            cache_dir = (self.cachePath(idict.get('cachepath'),
                                        idict.get('service_name')))
            self["cachepath"] = cache_dir
            self["req_cache_path"] = os.path.join(cache_dir, '.cache')
        self.setdefault("cert", None)
        self.setdefault("key", None)
        self.setdefault('capath', None)
        self.setdefault("timeout", 300)
        self.setdefault("logger", logging)

        check_server_url(self['host'])
Example #19
0
    def _postRequest(self, url, params, verb='POST', verbose=0):
        "Helper function to POST request to given URL"
        mgr = RequestHandler(logger=self.logger)
        headers = copy(self.configDict['headers'])
        headers.update({"Authorization": self._token})

        try:
            data = mgr.getdata(url,
                               params,
                               headers,
                               verb=verb,
                               verbose=verbose)
            return json.loads(data)
        except Exception as exc:
            self.logger.error("Failed to retrieve data from MonIT. Error: %s",
                              str(exc))
            return None
Example #20
0
class MicroServiceTest(unittest.TestCase):
    "Unit test for MicroService module"
    def setUp(self):
        "Setup MicroService for testing"
        self.app = ServiceManager()
        config = TestConfig
        manager = 'WMCore_t.Services_t.MicroService_t.MicroService_t.ServiceManager'
        config.views.data.manager = manager
        config.manager = manager
        mount = '/microservice'
        self.mgr = RequestHandler()
        self.port = config.main.port
        self.url = 'http://localhost:%s%s/data' % (self.port, mount)
        cherrypy.config["server.socket_port"] = self.port
        self.server = RestApiHub(self.app, config, mount)
        cherrypy.tree.mount(self.server, mount)
        cherrypy.engine.start()

    def tearDown(self):
        "Tear down MicroService"
        cherrypy.engine.exit()
        cherrypy.engine.stop()

    def postRequest(self, params):
        "Perform POST request to our MicroService"
        headers = {'Content-type': 'application/json'}
        print("### post call %s params=%s headers=%s" % (self.url, params, headers))
        data = self.mgr.getdata(self.url, params=params, headers=headers, \
                verb='POST', cert=cert(), ckey=ckey())
        print("### post call data %s" % data)
        return json.loads(data)

    def test_getState(self):
        "Test function for getting state of the MicroService"
        url = '%s/status' % self.url
        data = self.mgr.getdata(url, params={})
        state = "bla"
        data = {"request":{"state": state}}
        self.postRequest(data)
        data = self.mgr.getdata(url, params={})
        data = json.loads(data)
        print("### url=%s, data=%s" % (url, data))
        for row in data['result']:
            if 'state' in row:
                self.assertEqual(state, row['state'])
Example #21
0
    def setUp(self):
        "initialization"
        self.mgr = RequestHandler()
        self.ckey = os.path.join(os.environ['HOME'], '.globus/userkey.pem')
        self.cert = os.path.join(os.environ['HOME'], '.globus/usercert.pem')

        self.cricheader = 'Date: Tue, 06 Nov 2018 14:50:29 GMT\r\nServer: Apache/2.4.6 (CentOS) OpenSSL/1.0.2k-fips mod_wsgi/3.4 Python/2.7.5 mod_gridsite/2.3.4\r\nVary: Cookie\r\nX-Frame-Options: SAMEORIGIN\r\nSet-Cookie: sessionid=bc1xu8zi5rbbsd5fgjuklb2tk2r3f6tw; expires=Sun, 11-Nov-2018 14:50:29 GMT; httponly; Max-Age=432000; Path=/\r\nContent-Length: 32631\r\nContent-Type: application/json\r\n\r\n'
        self.dbsheader = 'Date: Tue, 06 Nov 2018 14:39:07 GMT\r\nServer: Apache\r\nCMS-Server-Time: D=1503 t=1541515147806112\r\nTransfer-Encoding: chunked\r\nContent-Type: text/html\r\n\r\n'
        self.HTTPheader = 'Date: Tue, 06 Nov 2018 14:50:29 GMT\r\nServer: Apache/2.4.6 (CentOS) OpenSSL/1.0.2k-fips mod_wsgi/3.4 Python/2.7.5 mod_gridsite/2.3.4\r\nVary: Cookie\r\nX-Frame-Options: SAMEORIGIN\r\nSet-Cookie: GRIDHTTP_PASSCODE=2c6da9c96efa2ad0farhda; domain=cms-cric.cern.ch; path=/; secure\r\nContent-Length: 32631\r\nContent-Type: application/json\r\n\r\n'
Example #22
0
def workqueueRequests(state=None):
    "Helper functions to get requests from WorkQueue"
    url = workqueueView('jobsByRequest')
    if state:
        pass  # we may need to use state when we'll query WorkQueue
    params = {}
    headers = {'Accept': 'application/json'}
    mgr = RequestHandler()
    data = mgr.getdata(url,
                       params=params,
                       headers=headers,
                       cert=cert(),
                       ckey=ckey())
    data = json.loads(data)
    rdict = {}
    for row in data.get('rows', []):
        rdict[row['key']] = row['value']
    return rdict
Example #23
0
def alterSubscription(phedexid, decision, comments, nodes=None):
    "Helper function to alter subscriptions for given phedex id and nodes"
    mgr = RequestHandler()
    headers = {'Accept': 'application/json'}
    nodes = nodes if nodes else getNodesForId(phedexid)
    params = {
        'decision': decision,
        'request': phedexid,
        'node': ','.join(nodes),
        'comments': comments
        }
    url = '%s/updaterequest'
    data = mgr.getdata(url, params, headers, ckey=ckey(), cert=cert(), verb='POST')
    result = json.loads(data)
    if not result:
        return False
    if 'already' in result:
        return True
    return result
Example #24
0
def renewRucioToken(rucioAuthUrl, userToken):
    """
    Provided a user Rucio token, check it's lifetime and extend it by another hour
    :param rucioAuthUrl: url to the rucio authentication server
    :param rucioAcct: rucio account to be used
    :return: a datetime.datetime object with the new token lifetime
    """
    params = {}
    headers = {"X-Rucio-Auth-Token": userToken}

    url = '%s/auth/validate' % rucioAuthUrl
    logging.info("Renewing the Rucio token...")
    mgr = RequestHandler()
    res = mgr.getdata(url, params=params, headers=headers, ckey=ckey(), cert=cert())
    try:
        newExpiration = eval(res)['lifetime']
    except Exception as exc:
        raise RuntimeError("Failed to renew Rucio token. Response: {} Error: {}".format(res, str(exc)))
    return newExpiration
Example #25
0
def getDataFromURL(url, proxyfilename = None):
    """
    Read the content of a URL and return it as a string.
    Type of content should not matter, it can be a json file or a tarball for example.

    url: the link you would like to retrieve
    proxyfilename: the x509 proxy certificate to be used in case auth is required

    Returns binary data encoded as a string, which can be later processed
    according to what kind of content it represents.
    """

    # Get rid of unicode which may cause problems in pycurl
    stringUrl = url.encode('ascii')

    reqHandler = RequestHandler()
    _, data = reqHandler.request(url=stringUrl, params={}, ckey=proxyfilename,
                                 cert=proxyfilename,
                                 capath=HTTPRequests.getCACertPath())

    return data
Example #26
0
def getDataFromURL(url, proxyfilename = None):
    """
    Read the content of a URL and return it as a string.
    Type of content should not matter, it can be a json file or a tarball for example.

    url: the link you would like to retrieve
    proxyfilename: the x509 proxy certificate to be used in case auth is required

    Returns binary data encoded as a string, which can be later processed
    according to what kind of content it represents.
    """

    # Get rid of unicode which may cause problems in pycurl
    stringUrl = url.encode('ascii')

    reqHandler = RequestHandler()
    _, data = reqHandler.request(url=stringUrl, params={}, ckey=proxyfilename,
                                 cert=proxyfilename,
                                 capath=HTTPRequests.getCACertPath())

    return data
Example #27
0
def agentsSites(url):
    "Return list of sites known in CMS WMAgents"
    sites_ready_in_agent = set()
    headers = {'Accept': 'application/json'}
    params = {}
    mgr = RequestHandler()
    res = mgr.getdata(url, params=params, headers=headers, ckey=ckey(), cert=cert())
    data = json.loads(res)
    agents = {}
    for r in [i['value'] for i in data['rows']]:
        team = r['agent_team']
        if team != 'production':
            continue
        agents.setdefault(team, []).append(r)
    for team, agents in agents.items():
        for agent in agents:
            if agent['status'] != 'ok':
                continue
            for site, sinfo in agent['WMBS_INFO']['thresholds'].iteritems():
                if sinfo['state'] in ['Normal']:
                    sites_ready_in_agent.add(site)
    return sites_ready_in_agent
Example #28
0
def agentsSites(url):
    "Return list of sites known in CMS WMAgents"
    sites_ready_in_agent = set()
    headers = {'Accept': 'application/json'}
    params = {}
    mgr = RequestHandler()
    res = mgr.getdata(url, params=params, headers=headers, ckey=ckey(), cert=cert())
    data = json.loads(res)
    agents = {}
    for r in [i['value'] for i in data['rows']]:
        team = r['agent_team']
        if team != 'production':
            continue
        agents.setdefault(team, []).append(r)
    for team, agents in agents.items():
        for agent in agents:
            if agent['status'] != 'ok':
                continue
            for site, sinfo in agent['WMBS_INFO']['thresholds'].iteritems():
                if sinfo['state'] in ['Normal']:
                    sites_ready_in_agent.add(site)
    return sites_ready_in_agent
Example #29
0
def alterSubscription(phedexid, decision, comments, nodes=None):
    "Helper function to alter subscriptions for given phedex id and nodes"
    mgr = RequestHandler()
    headers = {'Accept': 'application/json'}
    nodes = nodes if nodes else getNodesForId(phedexid)
    params = {
        'decision': decision,
        'request': phedexid,
        'node': ','.join(nodes),
        'comments': comments
    }
    url = '%s/updaterequest'
    data = mgr.getdata(url,
                       params,
                       headers,
                       ckey=ckey(),
                       cert=cert(),
                       verb='POST')
    result = json.loads(data)
    if not result:
        return False
    if 'already' in result:
        return True
    return result
Example #30
0
 def setUp(self):
     "Setup MicroService for testing"
     self.app = ServiceManager()
     config = TestConfig
     manager = 'WMCore_t.Services_t.MicroService_t.MicroService_t.ServiceManager'
     config.views.data.manager = manager
     config.manager = manager
     mount = '/microservice'
     self.mgr = RequestHandler()
     self.port = config.main.port
     self.url = 'http://localhost:%s%s/data' % (self.port, mount)
     cherrypy.config["server.socket_port"] = self.port
     self.server = RestApiHub(self.app, config, mount)
     cherrypy.tree.mount(self.server, mount)
     cherrypy.engine.start()
Example #31
0
def getRucioToken(rucioAuthUrl, rucioAcct):
    """
    Provided a Rucio account, fetch a token from the authentication server
    :param rucioAuthUrl: url to the rucio authentication server
    :param rucioAcct: rucio account to be used
    :return: an integer with the expiration time in EPOCH
    """
    params = {}
    headers = {"X-Rucio-Account": rucioAcct}

    url = '%s/auth/x509' % rucioAuthUrl
    logging.info("Requesting a token to Rucio for account: %s, against url: %s", rucioAcct, rucioAuthUrl)
    mgr = RequestHandler()
    res = mgr.getheader(url, params=params, headers=headers, ckey=ckey(), cert=cert())
    if res.getReason() == "OK":
        userToken = res.getHeaderKey('X-Rucio-Auth-Token')
        tokenExpiration = res.getHeaderKey('X-Rucio-Auth-Token-Expires')
        logging.info("Retrieved Rucio token valid until: %s", tokenExpiration)
        # convert the human readable expiration time to EPOCH time
        tokenExpiration = datetime.datetime.strptime(tokenExpiration, "%a, %d %b %Y %H:%M:%S %Z")
        tokenExpiration = int(tokenExpiration.strftime('%s'))
        return userToken, tokenExpiration

    raise RuntimeError("Failed to acquire a Rucio token. Error: {}".format(res.getReason()))
Example #32
0
    def __init__(self, url = 'http://localhost', idict=None):
        """
        url should really be host - TODO fix that when have sufficient code
        coverage and change _getURLOpener if needed
        """
        if  not idict:
            idict = {}
        dict.__init__(self, idict)
        self.pycurl = idict.get('pycurl', None)
        self.capath = idict.get('capath', None)
        if self.pycurl:
            self.reqmgr = RequestHandler()

        #set up defaults
        self.setdefault("accept_type", 'text/html')
        self.setdefault("content_type", 'application/x-www-form-urlencoded')
        self.additionalHeaders = {}

        # check for basic auth early, as if found this changes the url
        urlComponent = sanitizeURL(url)
        if urlComponent['username'] is not None:
            self.addBasicAuth(\
                urlComponent['username'], urlComponent['password'])
            url = urlComponent['url'] # remove user, password from url

        self.setdefault("host", url)

        # then update with the incoming dict
        self.update(idict)

        self['endpoint_components'] = urlparse.urlparse(self['host'])

        # If cachepath = None disable caching
        if 'cachepath' in idict and idict['cachepath'] is None:
            self["req_cache_path"] = None
        else:
            cache_dir = (self.cachePath(idict.get('cachepath'), \
                        idict.get('service_name')))
            self["cachepath"] = cache_dir
            self["req_cache_path"] = os.path.join(cache_dir, '.cache')
        self.setdefault("timeout", 300)
        self.setdefault("logger", logging)

        check_server_url(self['host'])
        # and then get the URL opener
        self.setdefault("conn", self._getURLOpener())
Example #33
0
class AuxCacheUpdateTasks(CherryPyPeriodicTask):
    """
    Updates Aux db update periodically. (i.e. TagCollector)
    """
    def __init__(self, rest, config):

        super(AuxCacheUpdateTasks, self).__init__(config)
        self.reqmgrAux = ReqMgrAux(config.reqmgr2_url, logger=self.logger)
        self.mgr = RequestHandler()

    def setConcurrentTasks(self, config):
        """
        sets the list of functions which
        """
        self.concurrentTasks = [{
            'func': self.updateAuxiliarDocs,
            'duration': config.tagCollectDuration
        }]

    def updateAuxiliarDocs(self, config):
        """
        Update the central couch database with auxiliary documents
        that need to be constanly updated whenever an update is
        made at the data source
        """
        self.logger.info("Updating auxiliary couch documents ...")

        self.reqmgrAux.populateCMSSWVersion(config.tagcollect_url,
                                            **config.tagcollect_args)

        try:
            data = self.mgr.getdata(config.unified_url,
                                    params={},
                                    headers={'Accept': 'application/json'})
            data = json.loads(data)
        except Exception as ex:
            msg = "Failed to retrieve unified configuration from github. Error: %s" % str(
                ex)
            msg += "\nRetrying again in the next cycle"
            self.logger.error(msg)
            return

        self.reqmgrAux.updateUnifiedConfig(data, docName="config")
class PyCurlManager(unittest.TestCase):
    """Test pycurl_manager module"""

    def setUp(self):
        "initialization"
        self.mgr = RequestHandler()
        self.ckey = os.path.join(os.environ['HOME'], '.globus/userkey.pem')
        self.cert = os.path.join(os.environ['HOME'], '.globus/usercert.pem')

    def testMulti(self):
        """
        Test fetch of several urls at once, one of the url relies on CERN SSO.
        """
        tfile = tempfile.NamedTemporaryFile()
        url1 = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader/help"
        url2 = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader/datatiers"
        url3 = "https://cms-gwmsmon.cern.ch/prodview/json/site_summary"
        cern_sso_cookie(url3, tfile.name, self.cert, self.ckey)
        cookie = {url3: tfile.name}
        urls = [url1, url2, url3]
        data = getdata(urls, self.ckey, self.cert, cookie=cookie)
        headers = 0
        for row in data:
            if '200 OK' in row['headers']:
                headers += 1
        self.assertTrue(headers, 3)

    def testSingle(self):
        """
        Test single call to CERN SSO url.
        """
        # test RequestHandler
        url = "https://cms-gwmsmon.cern.ch/prodview/json/site_summary"
        params = {}
        tfile = tempfile.NamedTemporaryFile()
        cern_sso_cookie(url, tfile.name, self.cert, self.ckey)
        cookie = {url: tfile.name}
        header, _ = self.mgr.request(url, params, cookie=cookie)
        self.assertTrue(header.status, 200)
Example #35
0
    def __init__(self, rest, config):

        super(AuxCacheUpdateTasks, self).__init__(config)
        self.reqmgrAux = ReqMgrAux(config.reqmgr2_url, logger=self.logger)
        self.mgr = RequestHandler()
Example #36
0
class MicroServiceTest(unittest.TestCase):
    "Unit test for MicroService module"

    def setUp(self):
        "Setup MicroService for testing"
        self.managerName = "ServiceManager"
        config = TestConfig
        manager = 'WMCore_t.MicroService_t.MicroService_t.%s' % self.managerName
        config.views.data.manager = manager
        config.manager = manager
        mount = '/microservice/data'
        self.mgr = RequestHandler()
        self.port = config.main.port
        self.url = 'http://localhost:%s%s' % (self.port, mount)
        cherrypy.config["server.socket_port"] = self.port
        self.app = ServiceManager(config)
        self.server = RestApiHub(self.app, config, mount)
        cherrypy.tree.mount(self.server, mount)
        cherrypy.engine.start()

    def tearDown(self):
        "Tear down MicroService"
        cherrypy.engine.stop()
        cherrypy.engine.exit()

    def postRequest(self, apiName, params):
        "Perform POST request to our MicroService"
        headers = {'Content-type': 'application/json'}
        url = self.url + "/%s" % apiName
        data = self.mgr.getdata(url, params=params, headers=headers, \
                                verb='POST', cert=cert(), ckey=ckey(), encode=True, decode=True)
        print("### post call data %s" % data)
        return data

    def testGetStatus(self):
        "Test function for getting state of the MicroService"
        api = "status"
        url = '%s/%s' % (self.url, api)
        params = {}
        data = self.mgr.getdata(url, params=params, encode=True, decode=True)
        self.assertEqual(data['result'][0]['microservice'], self.managerName)
        self.assertEqual(data['result'][0]['api'], api)

        params = {"service": "transferor"}
        data = self.mgr.getdata(url, params=params, encode=True, decode=True)
        self.assertEqual(data['result'][0]['microservice'], self.managerName)
        self.assertEqual(data['result'][0]['api'], api)

    def testGetInfo(self):
        "Test function for getting state of the MicroService"
        api = "status"
        url = '%s/%s' % (self.url, api)
        params = {}
        data = self.mgr.getdata(url, params=params, encode=True, decode=True)
        self.assertEqual(data['result'][0]['microservice'], self.managerName)
        self.assertEqual(data['result'][0]['api'], api)

        params = {"request": "fake_request_name"}
        data = self.mgr.getdata(url, params=params, encode=True, decode=True)
        self.assertEqual(data['result'][0]['microservice'], self.managerName)
        self.assertEqual(data['result'][0]['api'], api)

    def testPostCall(self):
        "Test function for getting state of the MicroService"
        api = "status"
        data = {"request": "fake_request_name"}
        data = self.postRequest(api, data)
        self.assertDictEqual(data['result'][0], {
            'status': 'OK',
            'api': 'info'
        })
Example #37
0
class MSRuleCleaner(MSCore):
    """
    MSRuleCleaner.py class provides the logic used to clean the Rucio
    block level data placement rules created by WMAgent.
    """
    def __init__(self, msConfig, logger=None):
        """
        Runs the basic setup and initialization for the MSRuleCleaner module
        :param msConfig: micro service configuration
        """
        super(MSRuleCleaner, self).__init__(msConfig, logger=logger)

        self.msConfig.setdefault("verbose", True)
        self.msConfig.setdefault("interval", 60)
        self.msConfig.setdefault("services", ['ruleCleaner'])
        self.msConfig.setdefault("rucioWmaAccount", "wma_test")
        self.msConfig.setdefault("rucioMStrAccount", "wmcore_transferor")
        self.msConfig.setdefault('enableRealMode', False)
        self.mode = "RealMode" if self.msConfig[
            'enableRealMode'] else "DryRunMode"
        self.emailAlert = EmailAlert(self.msConfig)
        self.curlMgr = RequestHandler()

        # Building all the Pipelines:
        pName = 'plineMSTrCont'
        self.plineMSTrCont = Pipeline(name=pName,
                                      funcLine=[
                                          Functor(self.setPlineMarker, pName),
                                          Functor(self.cleanRucioRules)
                                      ])
        pName = 'plineMSTrBlock'
        self.plineMSTrBlock = Pipeline(name=pName,
                                       funcLine=[
                                           Functor(self.setPlineMarker, pName),
                                           Functor(self.cleanRucioRules)
                                       ])
        pName = 'plineAgentCont'
        self.plineAgentCont = Pipeline(
            name=pName,
            funcLine=[
                Functor(self.setPlineMarker, pName),
                Functor(self.getRucioRules, 'container',
                        self.msConfig['rucioWmaAccount']),
                Functor(self.cleanRucioRules)
            ])
        pName = 'plineAgentBlock'
        self.plineAgentBlock = Pipeline(
            name=pName,
            funcLine=[
                Functor(self.setPlineMarker, pName),
                Functor(self.getRucioRules, 'block',
                        self.msConfig['rucioWmaAccount']),
                Functor(self.cleanRucioRules)
            ])
        pName = 'plineArchive'
        self.plineArchive = Pipeline(name=pName,
                                     funcLine=[
                                         Functor(self.setPlineMarker, pName),
                                         Functor(self.setClean),
                                         Functor(self.archive)
                                     ])

        # Building the different set of plines we will need later:
        # NOTE: The following are all the functional pipelines which are supposed to include
        #       a cleanup function and report cleanup status in the MSRuleCleanerWflow object
        self.cleanuplines = [
            self.plineMSTrCont, self.plineMSTrBlock, self.plineAgentCont,
            self.plineAgentBlock
        ]
        # Building an auxiliary list of cleanup pipeline names only:
        self.cleanupPipeNames = [pline.name for pline in self.cleanuplines]

        # Building lists of pipelines related only to Agents or MStransferror
        self.agentlines = [self.plineAgentCont, self.plineAgentBlock]
        self.mstrlines = [self.plineMSTrCont, self.plineMSTrBlock]

        # Initialization of the 'cleaned' and 'archived' counters:
        self.wfCounters = {'cleaned': {}, 'archived': 0}

    def resetCounters(self):
        """
        A simple function for zeroing the cleaned and archived counters.
        """
        for pline in self.cleanuplines:
            self.wfCounters['cleaned'][pline.name] = 0
        self.wfCounters['archived'] = 0

    def execute(self, reqStatus):
        """
        Executes the whole ruleCleaner logic
        :return: summary
        """
        # start threads in MSManager which should call this method
        summary = dict(RULECLEANER_REPORT)

        self.currThread = current_thread()
        self.currThreadIdent = self.currThread.name
        self.updateReportDict(summary, "thread_id", self.currThreadIdent)
        self.resetCounters()

        self.logger.info("MSRuleCleaner is running in mode: %s.", self.mode)

        # Build the list of workflows to work on:
        try:
            requestRecords = {}
            for status in reqStatus:
                requestRecords.update(self.getRequestRecords(status))
        except Exception as err:  # general error
            msg = "Unknown exception while fetching requests from ReqMgr2. Error: %s", str(
                err)
            self.logger.exception(msg)
            self.updateReportDict(summary, "error", msg)

        # Call _execute() and feed the relevant pipeline with the objects popped from requestRecords
        try:
            totalNumRequests, cleanNumRequests, archivedNumRequests = self._execute(
                requestRecords)
            msg = "\nNumber of processed workflows: %s."
            msg += "\nNumber of properly cleaned workflows: %s."
            msg += "\nNumber of archived workflows: %s."
            self.logger.info(msg, totalNumRequests, cleanNumRequests,
                             archivedNumRequests)
            self.updateReportDict(summary, "total_num_requests",
                                  totalNumRequests)
            self.updateReportDict(summary, "clean_num_requests",
                                  cleanNumRequests)
            self.updateReportDict(summary, "archived_num_requests",
                                  archivedNumRequests)
        except Exception as ex:
            msg = "Unknown exception while running MSRuleCleaner thread Error: %s"
            self.logger.exception(msg, str(ex))
            self.updateReportDict(summary, "error", msg)

        return summary

    def _execute(self, reqRecords):
        """
        Executes the MSRuleCleaner pipelines based on the workflow status
        :param reqList: A list of RequestRecords to work on
        :return:        a tuple with:
                            number of properly cleaned requests
                            number of processed workflows
                            number of archived workflows
        """
        # NOTE: The Input Cleanup, the Block Level Cleanup and the Archival
        #       Pipelines are executed sequentially in the above order.
        #       This way we assure ourselves that we archive only workflows
        #       that have accomplished the needed cleanup

        cleanNumRequests = 0
        totalNumRequests = 0

        # Call the workflow dispatcher:
        for _, req in reqRecords.items():
            wflow = MSRuleCleanerWflow(req)
            self._dispatchWflow(wflow)
            msg = "\n----------------------------------------------------------"
            msg += "\nMSRuleCleanerWflow: %s"
            msg += "\n----------------------------------------------------------"
            self.logger.debug(msg, pformat(wflow))
            totalNumRequests += 1
            if self._checkClean(wflow):
                cleanNumRequests += 1

        # Report the counters:
        for pline in self.cleanuplines:
            msg = "Workflows cleaned by pipeline: %s: %d"
            self.logger.info(msg, pline.name,
                             self.wfCounters['cleaned'][pline.name])
        archivedNumRequests = self.wfCounters['archived']
        self.logger.info("Workflows archived: %d", self.wfCounters['archived'])
        return totalNumRequests, cleanNumRequests, archivedNumRequests

    def _dispatchWflow(self, wflow):
        """
        A function intended to dispatch a workflow (e.g based on its status)
        through one or more functional pipelines in case there is some more
        complicated logic involved in the order we execute them but not just
        a sequentially
        """
        self.logger.debug("Dispatching workflow: %s", wflow['RequestName'])
        # NOTE: The following dispatch logic is a subject to be changed at any time

        # Resolve:
        # NOTE: First resolve any preliminary flags that will be needed further
        #       in the logic of the _dispatcher() itself
        if wflow['RequestStatus'] == 'announced':
            self.getMSOutputTransferInfo(wflow)

        # Clean:
        # Do not clean any Resubmission, but still let them be archived
        if wflow['RequestType'] == 'Resubmission':
            wflow['ForceArchive'] = True
            msg = "Skipping cleanup step for workflow: %s - RequestType is %s."
            msg += " Will try to archive it directly."
            self.logger.info(msg, wflow['RequestName'], wflow['RequestType'])
        elif wflow['RequestStatus'] in ['rejected', 'aborted-completed']:
            # NOTE: We do not check the ParentageResolved flag for these
            #       workflows, but we do need to clean output data placement
            #       rules from the agents for them
            for pline in self.agentlines:
                try:
                    pline.run(wflow)
                except Exception as ex:
                    msg = "%s: General error from pipeline. Workflow: %s. Error: \n%s. "
                    msg += "\nWill retry again in the next cycle."
                    self.logger.exception(msg, pline.name,
                                          wflow['RequestName'], str(ex))
                    continue
                if wflow['CleanupStatus'][pline.name]:
                    self.wfCounters['cleaned'][pline.name] += 1
        elif wflow['RequestStatus'] == 'announced' and not wflow[
                'ParentageResolved']:
            # NOTE: We skip workflows which are not having 'ParentageResolved'
            #       flag, but we still need some proper logging for them.
            msg = "Skipping workflow: %s - 'ParentageResolved' flag set to false."
            msg += " Will retry again in the next cycle."
            self.logger.info(msg, wflow['RequestName'])
        elif wflow[
                'RequestStatus'] == 'announced' and not wflow['TransferDone']:
            # NOTE: We skip workflows which have not yet finalised their TransferStatus
            #       in MSOutput, but we still need some proper logging for them.
            msg = "Skipping workflow: %s - 'TransferStatus' is 'pending' or 'TransferInfo' is missing in MSOutput."
            msg += " Will retry again in the next cycle."
            self.logger.info(msg, wflow['RequestName'])
        elif wflow['RequestStatus'] == 'announced':
            for pline in self.cleanuplines:
                try:
                    pline.run(wflow)
                except Exception as ex:
                    msg = "%s: General error from pipeline. Workflow: %s. Error:  \n%s. "
                    msg += "\nWill retry again in the next cycle."
                    self.logger.exception(msg, pline.name,
                                          wflow['RequestName'], str(ex))
                    continue
                if wflow['CleanupStatus'][pline.name]:
                    self.wfCounters['cleaned'][pline.name] += 1
        else:
            # We shouldn't be here:
            msg = "Skipping workflow: %s - "
            msg += "Does not fall under any of the defined categories."
            self.logger.error(msg, wflow['RequestName'])

        # Archive:
        try:
            self.plineArchive.run(wflow)
            self.wfCounters['archived'] += 1
        except MSRuleCleanerArchival as ex:
            msg = "%s: Archival Error: %s. "
            msg += " Will retry again in the next cycle."
            self.logger.error(msg, wflow['PlineMarkers'][-1], ex.message())
        except Exception as ex:
            msg = "%s General error from pipeline. Workflow: %s. Error: \n%s. "
            msg += "\nWill retry again in the next cycle."
            self.logger.exception(msg, wflow['PlineMarkers'][-1],
                                  wflow['RequestName'], str(ex))

    def setPlineMarker(self, wflow, pName):
        """
        A function intended to mark which is the pipeline currently working
        on the workflow. It is supposed to be called always as a first function
        in the pipeline.
        :param  wflow:   A MSRuleCleaner workflow representation
        :param  pName:   The name of the functional pipeline
        :return wflow:
        """
        # NOTE: The current functional pipeline MUST always be appended at the
        #       end of the 'PlineMarkers' list

        # First get rid of the default:
        if not wflow['PlineMarkers']:
            wflow['PlineMarkers'] = []

        # Then push our current value into the markers list:
        wflow['PlineMarkers'].append(pName)

        # Populate the list of flags to be used later:
        if pName not in wflow['RulesToClean']:
            if pName in self.cleanupPipeNames:
                wflow['RulesToClean'][pName] = []
        if pName not in wflow['CleanupStatus']:
            if pName in self.cleanupPipeNames:
                wflow['CleanupStatus'][pName] = False
        return wflow

    def _checkClean(self, wflow):
        """
        An auxiliary function used to only check the temporary cleanup status.
        It basically takes the pipelines registered in 'PlineMarkers' that have
        already worked on the workflow as a mask and applies this mask over
        the set of flags in the 'CleanupStatus' field and then reduces the
        result to a single bool value
        """
        # NOTE: This is one of the few functions taking a workflow as an argument
        #       but returning a bool, since it is an auxiliary function and is not
        #       supposed to be called as a standalone function in a pipeline.
        # NOTE: `all([]) == True`, ergo all the 'rejected' && 'aborted-completed' workflows
        #       are also counted as properly cleaned and can trigger archival later

        # Build a list of bool flags based on the mask of PlineMarkers
        cleanFlagsList = [
            wflow['CleanupStatus'][key] for key in wflow['PlineMarkers']
            if key in wflow['CleanupStatus'].keys()
        ]

        # If no one have worked on the workflow set the clean status to false
        if not wflow['PlineMarkers']:
            cleanStatus = False
        # If we have a mask longer than the list of flags avoid false positives
        # because of the behavior explained above - `all([]) == True`
        elif not cleanFlagsList:
            cleanStatus = False
        # Figure out the final value
        else:
            cleanStatus = all(cleanFlagsList)
        return cleanStatus

    def setClean(self, wflow):
        """
        A function to set the 'IsClean' flag based on the status from all the
        pipelines which have worked on the workflow (and have put their markers
        in the 'PlineMarkers' list)
        :param  wflow:      A MSRuleCleaner workflow representation
        :return wflow:
        """
        wflow['IsClean'] = self._checkClean(wflow)
        return wflow

    def archive(self, wflow):
        """
        Move the workflow to the proper archived status after checking
        the full cleanup status
        :param  wflow:      A MSRuleCleaner workflow representation
        :param  archStatus: Target status to transition after archival
        :return wflow:
        """
        # NOTE: check allowed status transitions with:
        #       https://github.com/dmwm/WMCore/blob/5961d2229b1e548e58259c06af154f33bce36c68/src/python/WMCore/ReqMgr/DataStructs/RequestStatus.py#L171
        if not (wflow['IsClean'] or wflow['ForceArchive']):
            msg = "Not properly cleaned workflow: %s" % wflow['RequestName']
            raise MSRuleCleanerArchival(msg)

        # Check the available status transitions before we decide the final status
        targetStatusList = RequestStatus.REQUEST_STATE_TRANSITION.get(
            wflow['RequestStatus'], [])
        self.logger.info("targetStatusList: %s", targetStatusList)
        return wflow

    def getMSOutputTransferInfo(self, wflow):
        """
        Fetches the transfer information from the MSOutput REST interface for
        the given workflow.
        :param  wflow:   A MSRuleCleaner workflow representation
        :return wflow:
        """
        headers = {'Accept': 'application/json'}
        params = {}
        url = '%s/data/info?request=%s' % (self.msConfig['msOutputUrl'],
                                           wflow['RequestName'])
        try:
            res = self.curlMgr.getdata(url,
                                       params=params,
                                       headers=headers,
                                       ckey=ckey(),
                                       cert=cert())
            data = json.loads(res)['result'][0]
            transferInfo = data['transferDoc']
        except Exception as ex:
            msg = "General exception while fetching TransferInfo from MSOutput for %s. "
            msg += "Error: %s"
            self.logger.exception(msg, wflow['RequestName'], str(ex))
        if transferInfo is not None and transferInfo[
                'TransferStatus'] == 'done':
            wflow['TransferDone'] = True
        return wflow

    def getRucioRules(self, wflow, gran, rucioAcct):
        """
        Queries Rucio and builds the relevant list of blocklevel rules for
        the given workflow
        :param  wflow:   A MSRuleCleaner workflow representation
        :param  gran:    Data granularity to search for Rucio rules. Possible values:
                        'block' || 'container'
        :return:        wflow
        """
        currPline = wflow['PlineMarkers'][-1]
        # Find all the output placement rules created by the agents
        for dataCont in wflow['OutputDatasets']:
            if gran == 'container':
                for rule in self.rucio.listDataRules(dataCont,
                                                     account=rucioAcct):
                    wflow['RulesToClean'][currPline].append(rule['id'])
            elif gran == 'block':
                try:
                    blocks = self.rucio.getBlocksInContainer(dataCont)
                    for block in blocks:
                        for rule in self.rucio.listDataRules(
                                block, account=rucioAcct):
                            wflow['RulesToClean'][currPline].append(rule['id'])
                except WMRucioDIDNotFoundException:
                    msg = "Container: %s not found in Rucio for workflow: %s."
                    self.logger.info(msg, dataCont, wflow['RequestName'])
        return wflow

    def cleanRucioRules(self, wflow):
        """
        Cleans all the Rules present in the field 'RulesToClean' in the MSRuleCleaner
        workflow representation. And fills the relevant Cleanup Status.
        :param wflow:   A MSRuleCleaner workflow representation
        :return:        wflow
        """
        # NOTE: The function should be called independently and sequentially from
        #       The Input and the respective BlockLevel pipelines.

        # NOTE: The current functional pipeline is always the last one in the PlineMarkers list
        currPline = wflow['PlineMarkers'][-1]
        delResults = []
        if self.msConfig['enableRealMode']:
            for rule in wflow['RulesToClean'][currPline]:
                self.logger.info("%s: Deleting ruleId: %s ", currPline, rule)
                delResult = self.rucio.deleteRule(rule)
                delResults.append(delResult)
                if not delResult:
                    self.logger.warning("%s: Failed to delete ruleId: %s ",
                                        currPline, rule)
        else:
            for rule in wflow['RulesToClean'][currPline]:
                delResults.append(True)
                self.logger.info("%s: DRY-RUN: Is about to delete ruleId: %s ",
                                 currPline, rule)

        # Set the cleanup flag:
        wflow['CleanupStatus'][currPline] = all(delResults)
        # ----------------------------------------------------------------------
        # FIXME : To be removed once the plineMSTrBlock && plineMSTrCont are
        #         developed
        if wflow['CleanupStatus'][currPline] in [
                'plineMSTrBlock', 'plineMSTrCont'
        ]:
            wflow['CleanupStatus'][currPline] = True
        # ----------------------------------------------------------------------
        return wflow

    def getRequestRecords(self, reqStatus):
        """
        Queries ReqMgr2 for requests in a given status.
        :param reqStatus: The status for the requests to be fetched from ReqMgr2
        :return requests: A dictionary with all the workflows in the given status
        """
        self.logger.info("Fetching requests in status: %s", reqStatus)
        result = self.reqmgr2.getRequestByStatus([reqStatus], detail=True)
        if not result:
            requests = {}
        else:
            requests = result[0]
        self.logger.info('  retrieved %s requests in status: %s',
                         len(requests), reqStatus)
        return requests
Example #38
0
    def __init__(self, config, quiet):
        """
        Initialise class members
        """
        self.config = config.General
        self.max_files_per_block = self.config.max_files_per_block
        #self.userCert = self.config.opsCert
        #self.userKey = self.config.opsKey
        self.block_publication_timeout = self.config.block_closure_timeout
        self.lfn_map = {}
        self.force_publication = False
        self.force_failure = False
        #TODO: logger!
        def createLogdir(dirname):
            """ Create the directory dirname ignoring erors in case it exists. Exit if
                the directory cannot be created.
            """
            try:
                os.mkdir(dirname)
            except OSError as ose:
                if ose.errno != 17: #ignore the "Directory already exists error"
                    print(str(ose))
                    print("The task worker need to access the '%s' directory" % dirname)
                    sys.exit(1)


        def setRootLogger(quiet, debug):
            """Sets the root logger with the desired verbosity level
               The root logger logs to logs/twlog.txt and every single
               logging instruction is propagated to it (not really nice
               to read)

            :arg bool quiet: it tells if a quiet logger is needed
            :arg bool debug: it tells if needs a verbose logger
            :return logger: a logger with the appropriate logger level."""

            createLogdir('logs')
            createLogdir('logs/processes')
            createLogdir('logs/tasks')

            logHandler = MultiProcessingLog('logs/log.txt', when='midnight')
            logFormatter = logging.Formatter("%(asctime)s:%(levelname)s:%(module)s,%(lineno)d:%(message)s")
            logHandler.setFormatter(logFormatter)
            logging.getLogger().addHandler(logHandler)
            loglevel = logging.INFO
            if quiet:
                loglevel = logging.WARNING
            if debug:
                loglevel = logging.DEBUG
            logging.getLogger().setLevel(loglevel)
            logger = setProcessLogger("master")
            logger.debug("PID %s.", os.getpid())
            logger.debug("Logging level initialized to %s.", loglevel)
            return logger

        self.cache_area = self.config.cache_area
        self.logger = setRootLogger(quiet, True)

        try:
            self.oracleDB = HTTPRequests(self.config.oracleDB,
                                         self.config.opsCert,
                                         self.config.opsKey)
            self.logger.debug('Contacting OracleDB:' + self.config.oracleDB)
        except:
            self.logger.exception('Failed when contacting Oracle')
            raise

        try:
            self.connection = RequestHandler(config={'timeout': 900, 'connecttimeout' : 900})
        except Exception as ex:
            msg = "Error initializing the connection"
            msg += str(ex)
            msg += str(traceback.format_exc())
            self.logger.debug(msg)
Example #39
0
class AlertManagerAPI(object):
    """
    A class used to send alerts via the MONIT AlertManager API
    """
    def __init__(self, alertManagerUrl, logger=None):
        self.alertManagerUrl = alertManagerUrl
        # sender's hostname is added as an annotation
        self.hostname = socket.gethostname()
        self.mgr = RequestHandler()
        self.ltz = LocalTimezone()
        self.headers = {"Content-Type": "application/json"}
        self.validSeverity = ["high", "medium", "low"]
        self.logger = logger if logger else logging.getLogger()

    def sendAlert(self,
                  alertName,
                  severity,
                  summary,
                  description,
                  service,
                  tag="wmcore",
                  endSecs=600,
                  generatorURL=""):
        """
        :param alertName: a unique name for the alert
        :param severity: low, medium, high
        :param summary: a short description of the alert
        :param description: a longer informational message with details about the alert
        :param service: the name of the service firing an alert
        :param tag: a unique tag used to help route the alert
        :param endSecs: how many minutes until the alarm is silenced
        :param generatorURL: this URL will be sent to AlertManager and configured as a clickable "Source" link in the web interface

        AlertManager JSON format reference: https://www.prometheus.io/docs/alerting/latest/clients/
        [
          {
            "labels": {
            "alertname": "<requiredAlertName>",
            "<labelname>": "<labelvalue>",
            ...
          },
            "annotations": {
              "<labelname>": "<labelvalue>",
            ...
          },
            "startsAt": "<rfc3339>", # optional, will be current time if not present
            "endsAt": "<rfc3339>",
            "generatorURL": "<generator_url>" # optional
          },
        ]
        """

        if not self._isValidSeverity(severity):
            return False

        request = []
        alert = {}
        labels = {}
        annotations = {}

        # add labels
        labels["alertname"] = alertName
        labels["severity"] = severity
        labels["tag"] = tag
        labels["service"] = service
        alert["labels"] = labels

        # add annotations
        annotations["hostname"] = self.hostname
        annotations["summary"] = summary
        annotations["description"] = description
        alert["annotations"] = annotations

        # In python3 we won't need the LocalTimezone class
        # Will change to d = datetime.now().astimezone() + timedelta(seconds=endSecs)
        d = datetime.now(self.ltz) + timedelta(seconds=endSecs)
        alert["endsAt"] = d.isoformat("T")
        alert["generatorURL"] = generatorURL

        request.append(alert)
        # need to do this because pycurl_manager only accepts dict and encoded strings type
        params = json.dumps(request)

        res = self.mgr.getdata(self.alertManagerUrl,
                               params=params,
                               headers=self.headers,
                               verb='POST')

        return res

    def _isValidSeverity(self, severity):
        """
        Used to check if the severity of the alert matches the valid levels: low, medium, high
        :param severity: severity of the alert
        :return: True or False
        """
        if severity not in self.validSeverity:
            logging.critical(
                "Alert submitted to AlertManagerAPI with invalid severity: %s",
                severity)
            return False
        return True
Example #40
0
class Requests(dict):
    """
    Generic class for sending different types of HTTP Request to a given URL
    """
    def __init__(self, url='http://localhost', idict=None):
        """
        url should really be host - TODO fix that when have sufficient code
        coverage and change _getURLOpener if needed
        """
        if not idict:
            idict = {}
        dict.__init__(self, idict)
        self.pycurl = idict.get('pycurl', None)
        self.capath = idict.get('capath', None)
        if self.pycurl:
            self.reqmgr = RequestHandler()

        #set up defaults
        self.setdefault("accept_type", 'text/html')
        self.setdefault("content_type", 'application/x-www-form-urlencoded')
        self.additionalHeaders = {}

        # check for basic auth early, as if found this changes the url
        urlComponent = sanitizeURL(url)
        if urlComponent['username'] is not None:
            self.addBasicAuth(\
                urlComponent['username'], urlComponent['password'])
            url = urlComponent['url']  # remove user, password from url

        self.setdefault("host", url)

        # then update with the incoming dict
        self.update(idict)

        self['endpoint_components'] = urlparse.urlparse(self['host'])

        # If cachepath = None disable caching
        if 'cachepath' in idict and idict['cachepath'] is None:
            self["req_cache_path"] = None
        else:
            cache_dir = (self.cachePath(idict.get('cachepath'), \
                        idict.get('service_name')))
            self["cachepath"] = cache_dir
            self["req_cache_path"] = os.path.join(cache_dir, '.cache')
        self.setdefault("timeout", 300)
        self.setdefault("logger", logging)

        check_server_url(self['host'])
        # and then get the URL opener
        self.setdefault("conn", self._getURLOpener())

    def get(self,
            uri=None,
            data={},
            incoming_headers={},
            encode=True,
            decode=True,
            contentType=None):
        """
        GET some data
        """
        return self.makeRequest(uri, data, 'GET', incoming_headers, encode,
                                decode, contentType)

    def post(self,
             uri=None,
             data={},
             incoming_headers={},
             encode=True,
             decode=True,
             contentType=None):
        """
        POST some data
        """
        return self.makeRequest(uri, data, 'POST', incoming_headers, encode,
                                decode, contentType)

    def put(self,
            uri=None,
            data={},
            incoming_headers={},
            encode=True,
            decode=True,
            contentType=None):
        """
        PUT some data
        """
        return self.makeRequest(uri, data, 'PUT', incoming_headers, encode,
                                decode, contentType)

    def delete(self,
               uri=None,
               data={},
               incoming_headers={},
               encode=True,
               decode=True,
               contentType=None):
        """
        DELETE some data
        """
        return self.makeRequest(uri, data, 'DELETE', incoming_headers, encode,
                                decode, contentType)

    def makeRequest(self,
                    uri=None,
                    data={},
                    verb='GET',
                    incoming_headers={},
                    encoder=True,
                    decoder=True,
                    contentType=None):
        """
        Wrapper around request helper functions.
        """
        if self.pycurl:
            result = self.makeRequest_pycurl(uri, data, verb, incoming_headers,
                                             encoder, decoder, contentType)
        else:
            result = self.makeRequest_httplib(uri, data, verb,
                                              incoming_headers, encoder,
                                              decoder, contentType)
        return result

    def makeRequest_pycurl(self,
                           uri=None,
                           params={},
                           verb='GET',
                           incoming_headers={},
                           encoder=True,
                           decoder=True,
                           contentType=None):
        """
        Make HTTP(s) request via pycurl library. Stay complaint with
        makeRequest_httplib method.
        """
        ckey, cert = self.getKeyCert()
        capath = self.getCAPath()
        if not contentType:
            contentType = self['content_type']
        headers = {
            "Content-type": contentType,
            "User-agent": "WMCore.Services.Requests/v001",
            "Accept": self['accept_type']
        }
        for key in self.additionalHeaders.keys():
            headers[key] = self.additionalHeaders[key]
        #And now overwrite any headers that have been passed into the call:
        headers.update(incoming_headers)
        url = self['host'] + uri
        response, data = self.reqmgr.request(url, params, headers, \
                    verb=verb, ckey=ckey, cert=cert, capath=capath, decode=decoder)
        return data, response.status, response.reason, response.fromcache

    def makeRequest_httplib(self,
                            uri=None,
                            data={},
                            verb='GET',
                            incoming_headers={},
                            encoder=True,
                            decoder=True,
                            contentType=None):
        """
        Make a request to the remote database. for a give URI. The type of
        request will determine the action take by the server (be careful with
        DELETE!). Data should be a dictionary of {dataname: datavalue}.

        Returns a tuple of the data from the server, decoded using the
        appropriate method the response status and the response reason, to be
        used in error handling.

        You can override the method to encode/decode your data by passing in an
        encoding/decoding function to this method. Your encoded data must end up
        as a string.

        """
        #TODO: User agent should be:
        # $client/$client_version (CMS)
        # $http_lib/$http_lib_version $os/$os_version ($arch)
        if not contentType:
            contentType = self['content_type']
        headers = {
            "Content-type": contentType,
            "User-agent": "WMCore.Services.Requests/v001",
            "Accept": self['accept_type']
        }
        encoded_data = ''

        for key in self.additionalHeaders.keys():
            headers[key] = self.additionalHeaders[key]

        #And now overwrite any headers that have been passed into the call:
        #WARNING: doesn't work with deplate so only accept gzip
        incoming_headers["accept-encoding"] = "gzip,identity"
        headers.update(incoming_headers)

        # httpib2 requires absolute url
        uri = self['host'] + uri

        # If you're posting an attachment, the data might not be a dict
        #   please test against ConfigCache_t if you're unsure.
        #assert type(data) == type({}), \
        #        "makeRequest input data must be a dict (key/value pairs)"

        # There must be a better way to do this...
        def f():
            """Dummy function"""
            pass

        if verb != 'GET' and data:
            if type(encoder) == type(self.get) or type(encoder) == type(f):
                encoded_data = encoder(data)
            elif encoder == False:
                # Don't encode the data more than we have to
                #  we don't want to URL encode the data blindly,
                #  that breaks POSTing attachments... ConfigCache_t
                #encoded_data = urllib.urlencode(data)
                #  -- Andrew Melo 25/7/09
                encoded_data = data
            else:
                # Either the encoder is set to True or it's junk, so use
                # self.encode
                encoded_data = self.encode(data)
            headers["Content-length"] = len(encoded_data)
        elif verb == 'GET' and data:
            #encode the data as a get string
            uri = "%s?%s" % (uri, urllib.urlencode(data, doseq=True))

        headers["Content-length"] = str(len(encoded_data))

        assert type(encoded_data) == type('string'), \
            "Data in makeRequest is %s and not encoded to a string" \
                % type(encoded_data)

        # httplib2 will allow sockets to close on remote end without retrying
        # try to send request - if this fails try again - should then succeed
        try:
            response, result = self['conn'].request(uri,
                                                    method=verb,
                                                    body=encoded_data,
                                                    headers=headers)
            if response.status == 408:  # timeout can indicate a socket error
                response, result = self['conn'].request(uri,
                                                        method=verb,
                                                        body=encoded_data,
                                                        headers=headers)
        except (socket.error, AttributeError):
            # AttributeError implies initial connection error - need to close
            # & retry. httplib2 doesn't clear httplib state before next request
            # if this is threaded this may spoil things
            # only have one endpoint so don't need to determine which to shut
            [conn.close() for conn in self['conn'].connections.values()]
            self['conn'] = self._getURLOpener()
            # ... try again... if this fails propagate error to client
            try:
                response, result = self['conn'].request(uri,
                                                        method=verb,
                                                        body=encoded_data,
                                                        headers=headers)
            except AttributeError:
                # socket/httplib really screwed up - nuclear option
                self['conn'].connections = {}
                raise socket.error, 'Error contacting: %s' \
                        % self.getDomainName()
        if response.status >= 400:
            e = HTTPException()
            setattr(e, 'req_data', encoded_data)
            setattr(e, 'req_headers', headers)
            setattr(e, 'url', uri)
            setattr(e, 'result', result)
            setattr(e, 'status', response.status)
            setattr(e, 'reason', response.reason)
            setattr(e, 'headers', response)
            raise e

        if type(decoder) == type(self.makeRequest) or type(decoder) == type(f):
            result = decoder(result)
        elif decoder != False:
            result = self.decode(result)
        #TODO: maybe just return result and response...
        return result, response.status, response.reason, response.fromcache

    def encode(self, data):
        """
        encode data into some appropriate format, for now make it a string...
        """
        return urllib.urlencode(data, doseq=1)

    def decode(self, data):
        """
        decode data to some appropriate format, for now make it a string...
        """
        return data.__str__()

    def cachePath(self, given_path, service_name):
        """Return cache location"""
        if not service_name:
            service_name = 'REQUESTS'
        top = self.cacheTopPath(given_path, service_name)

        # deal with multiple Services that have the same service running and
        # with multiple users for a given Service
        if self.getUserName() is None:
            cachepath = os.path.join(top, self['endpoint_components'].netloc)
        else:
            cachepath = os.path.join(top, '%s-%s' \
                % (self.getUserName(), self.getDomainName()))

        try:
            # only we should be able to write to this dir
            os.makedirs(cachepath, stat.S_IRWXU)
        except OSError:
            if not os.path.isdir(cachepath):
                raise
            Permissions.owner_readwriteexec(cachepath)

        return cachepath

    def cacheTopPath(self, given_path, service_name):
        """Where to cache results?

        Logic:
          o If passed in take that
          o Is the environment variable "SERVICE_NAME"_CACHE_DIR defined?
          o Is WMCORE_CACHE_DIR set
          o Generate a temporary directory
          """
        if given_path:
            return given_path
        user = str(os.getuid())
        # append user id so users don't clobber each other
        lastbit = os.path.join('.wmcore_cache_%s' % user, service_name.lower())
        for var in ('%s_CACHE_DIR' % service_name.upper(), 'WMCORE_CACHE_DIR'):
            if os.environ.get(var):
                firstbit = os.environ[var]
                break
        else:
            idir = tempfile.mkdtemp(prefix='.wmcore_cache_')
            # object to store temporary directory - cleaned up on destruction
            self['deleteCacheOnExit'] = TempDirectory(idir)
            return idir

        return os.path.join(firstbit, lastbit)

    def getDomainName(self):
        """Parse netloc info to get hostname"""
        return self['endpoint_components'].hostname

    def getUserName(self):
        """Parse netloc to get user"""
        return self['endpoint_components'].username

    def _getURLOpener(self):
        """
        method getting a secure (HTTPS) connection
        """
        key, cert = None, None
        if self['endpoint_components'].scheme == 'https':
            # only add certs to https requests
            # if we have a key/cert add to request,
            # if not proceed as not all https connections require them
            try:
                key, cert = self.getKeyCert()
            except Exception, ex:
                msg = 'No certificate or key found, authentication may fail'
                self['logger'].info(msg)
                self['logger'].debug(str(ex))

        try:
            # disable validation as we don't have a single PEM with all ca's
            http = httplib2.Http(self['req_cache_path'],
                                 self['timeout'],
                                 disable_ssl_certificate_validation=True)
        except TypeError:
            # old httplib2 versions disable validation by default
            http = httplib2.Http(self['req_cache_path'], self['timeout'])

        # Domain must be just a hostname and port. self[host] is a URL currently
        if key or cert:
            http.add_certificate(key=key, cert=cert, domain='')
        return http
Example #41
0
class Worker(object):
    """

    """
    def __init__(self, config, quiet):
        """
        Initialise class members
        """
        self.config = config.General
        self.max_files_per_block = self.config.max_files_per_block
        self.userProxy = self.config.opsProxy
        self.block_publication_timeout = self.config.block_closure_timeout
        self.lfn_map = {}
        self.force_publication = False
        self.force_failure = False
        #TODO: logger!
        def createLogdir(dirname):
            """ Create the directory dirname ignoring erors in case it exists. Exit if
                the directory cannot be created.
            """
            try:
                os.mkdir(dirname)
            except OSError as ose:
                if ose.errno != 17: #ignore the "Directory already exists error"
                    print(str(ose))
                    print("The task worker need to access the '%s' directory" % dirname)
                    sys.exit(1)


        def setRootLogger(quiet, debug):
            """Sets the root logger with the desired verbosity level
               The root logger logs to logs/twlog.txt and every single
               logging instruction is propagated to it (not really nice
               to read)

            :arg bool quiet: it tells if a quiet logger is needed
            :arg bool debug: it tells if needs a verbose logger
            :return logger: a logger with the appropriate logger level."""

            createLogdir('logs')
            createLogdir('logs/processes')
            createLogdir('logs/tasks')

            logHandler = MultiProcessingLog('logs/log.txt', when='midnight')
            logFormatter = logging.Formatter("%(asctime)s:%(levelname)s:%(module)s,%(lineno)d:%(message)s")
            logHandler.setFormatter(logFormatter)
            logging.getLogger().addHandler(logHandler)
            loglevel = logging.INFO
            if quiet:
                loglevel = logging.WARNING
            if debug:
                loglevel = logging.DEBUG
            logging.getLogger().setLevel(loglevel)
            logger = setProcessLogger("master")
            logger.debug("PID %s.", os.getpid())
            logger.debug("Logging level initialized to %s.", loglevel)
            return logger

        self.cache_area = self.config.cache_area
        self.logger = setRootLogger(quiet, True)

        try:
            self.oracleDB = HTTPRequests(self.config.oracleDB,
                                         self.config.opsProxy,
                                         self.config.opsProxy)
            self.logger.debug('Contacting OracleDB:' + self.config.oracleDB)
        except:
            self.logger.exception('Failed when contacting Oracle')
            raise

        try:
            self.connection = RequestHandler(config={'timeout': 900, 'connecttimeout' : 900})
        except Exception as ex:
            msg = "Error initializing the connection"
            msg += str(ex)
            msg += str(traceback.format_exc())
            self.logger.debug(msg)


    def active_tasks(self, db):

        fileDoc = {}
        fileDoc['asoworker'] = self.config.asoworker
        fileDoc['subresource'] = 'acquirePublication'

        self.logger.debug("Retrieving publications from oracleDB")

        results = ''
        try:
            results = db.post(self.config.oracleFileTrans,
                              data=encodeRequest(fileDoc))
        except Exception as ex:
            self.logger.error("Failed to acquire publications \
                                from oracleDB: %s" %ex)
            return []

        fileDoc = dict()
        fileDoc['asoworker'] = self.config.asoworker
        fileDoc['subresource'] = 'acquiredPublication'
        fileDoc['grouping'] = 0
        fileDoc['limit'] = 100000

        self.logger.debug("Retrieving max.100000 acquired puclications from oracleDB")

        result = []

        try:
            results = db.get(self.config.oracleFileTrans,
                             data=encodeRequest(fileDoc))
            result.extend(oracleOutputMapping(results))
        except Exception as ex:
            self.logger.error("Failed to acquire publications \
                                from oracleDB: %s" %ex)
            return []

        self.logger.debug("publen: %s" % len(result))

        self.logger.debug("%s acquired puclications retrieved" % len(result))
        #TODO: join query for publisher (same of submitter)
        unique_tasks = [list(i) for i in set(tuple([x['username'],
                                                    x['user_group'],
                                                    x['user_role'],
                                                    x['taskname']]
                                                  ) for x in result if x['transfer_state'] == 3)]

        info = []
        for task in unique_tasks:
            info.append([x for x in result if x['taskname'] == task[3]])
        return zip(unique_tasks, info)

    def getPublDescFiles(self, workflow, lfn_ready):
        """
        Download and read the files describing
        what needs to be published
        """
        data = {}
        data['taskname'] = workflow
        data['filetype'] = 'EDM'

        out = []
        # divide lfn per chunks, avoiding URI-too long exception
        def chunks(l, n):
            """
            Yield successive n-sized chunks from l.
            :param l: list to splitt in chunks
            :param n: chunk size
            :return: yield the next list chunk
            """
            for i in range(0, len(l), n):
                yield l[i:i + n]

        for  lfn_ in chunks(lfn_ready, 50):
            data['lfn'] = lfn_

            try:
                res = self.oracleDB.get('/crabserver/dev/filemetadata',
                                        data=encodeRequest(data, listParams=["lfn"]))
                res = res[0]
            except Exception as ex:
                self.logger.error("Error during metadata retrieving: %s" %ex)

            print(len(res['result']))
            for obj in res['result']:
                if isinstance(obj, dict):
                    out.append(obj)
                else:
                    #print type(obj)
                    out.append(json.loads(str(obj)))

        return out

    def algorithm(self):
        """
        1. Get a list of users with files to publish from the couchdb instance
        2. For each user get a suitably sized input for publish
        3. Submit the publish to a subprocess
        """
        tasks = self.active_tasks(self.oracleDB)

        self.logger.debug('kicking off pool %s' % [x[0][3] for x in tasks])
        processes = []

        try:
            for task in tasks:
                p = Process(target=self.startSlave, args=(task,))
                p.start()
                processes.append(p)

            for proc in processes:
                proc.join()
        except:
            self.logger.exception("Error during process mapping")


    def startSlave(self, task):
        # TODO: lock task!
        # - process logger
        logger = setProcessLogger(str(task[0][3]))
        logger.info("Process %s is starting. PID %s", task[0][3], os.getpid())

        self.force_publication = False
        workflow = str(task[0][3])
        wfnamemsg = "%s: " % (workflow)

        if len(task[1]) > self.max_files_per_block:
            self.force_publication = True
            msg = "All datasets have more than %s ready files." % (self.max_files_per_block)
            msg += " No need to retrieve task status nor last publication time."
            logger.info(wfnamemsg+msg)
        else:
            msg = "At least one dataset has less than %s ready files." % (self.max_files_per_block)
            logger.info(wfnamemsg+msg)
            # Retrieve the workflow status. If the status can not be retrieved, continue
            # with the next workflow.
            workflow_status = ''
            url = '/'.join(self.cache_area.split('/')[:-1]) + '/workflow'
            msg = "Retrieving status from %s" % (url)
            logger.info(wfnamemsg+msg)
            buf = cStringIO.StringIO()
            header = {"Content-Type":"application/json"}
            data = {'workflow': workflow}#, 'subresource': 'taskads'}
            try:
                _, res_ = self.connection.request(url,
                                                  data,
                                                  header,
                                                  doseq=True,
                                                  ckey=self.userProxy,
                                                  cert=self.userProxy
                                                 )# , verbose=True) #  for debug
            except Exception as ex:
                if self.config.isOracle:
                    logger.exception('Error retrieving status from cache.')
                    return 0

            msg = "Status retrieved from cache. Loading task status."
            logger.info(wfnamemsg+msg)
            try:
                buf.close()
                res = json.loads(res_)
                workflow_status = res['result'][0]['status']
                msg = "Task status is %s." % workflow_status
                logger.info(wfnamemsg+msg)
            except ValueError:
                msg = "Workflow removed from WM."
                logger.error(wfnamemsg+msg)
                workflow_status = 'REMOVED'
            except Exception as ex:
                msg = "Error loading task status!"
                msg += str(ex)
                msg += str(traceback.format_exc())
                logger.error(wfnamemsg+msg)
            # If the workflow status is terminal, go ahead and publish all the ready files
            # in the workflow.
            if workflow_status in ['COMPLETED', 'FAILED', 'KILLED', 'REMOVED']:
                self.force_publication = True
                if workflow_status in ['KILLED', 'REMOVED']:
                    self.force_failure = True
                msg = "Considering task status as terminal. Will force publication."
                logger.info(wfnamemsg+msg)
            # Otherwise...
            else:
                msg = "Task status is not considered terminal."
                logger.info(wfnamemsg+msg)
                msg = "Getting last publication time."
                logger.info(wfnamemsg+msg)
                # Get when was the last time a publication was done for this workflow (this
                # should be more or less independent of the output dataset in case there are
                # more than one).
                last_publication_time = None
                data = {}
                data['workflow'] = workflow
                data['subresource'] = 'search'
                try:
                    result = self.oracleDB.get(self.config.oracleFileTrans.replace('filetransfers', 'task'),
                                               data=encodeRequest(data))
                    logger.debug("task: %s " %  str(result[0]))
                    logger.debug("task: %s " %  getColumn(result[0], 'tm_last_publication'))
                except Exception as ex:
                    logger.error("Error during task doc retrieving: %s" %ex)
                if last_publication_time:
                    date = oracleOutputMapping(result)['last_publication']
                    seconds = datetime.strptime(date, "%Y-%m-%d %H:%M:%S.%f").timetuple()
                    last_publication_time = time.mktime(seconds)

                msg = "Last publication time: %s." % str(last_publication_time)
                logger.debug(wfnamemsg+msg)
                # If this is the first time a publication would be done for this workflow, go
                # ahead and publish.
                if not last_publication_time:
                    self.force_publication = True
                    msg = "There was no previous publication. Will force publication."
                    logger.info(wfnamemsg+msg)
                # Otherwise...
                else:
                    last = last_publication_time
                    msg = "Last published block: %s" % (last)
                    logger.debug(wfnamemsg+msg)
                    # If the last publication was long time ago (> our block publication timeout),
                    # go ahead and publish.
                    now = int(time.time()) - time.timezone
                    time_since_last_publication = now - last
                    hours = int(time_since_last_publication/60/60)
                    minutes = int((time_since_last_publication - hours*60*60)/60)
                    timeout_hours = int(self.block_publication_timeout/60/60)
                    timeout_minutes = int((self.block_publication_timeout - timeout_hours*60*60)/60)
                    msg = "Last publication was %sh:%sm ago" % (hours, minutes)
                    if time_since_last_publication > self.block_publication_timeout:
                        self.force_publication = True
                        msg += " (more than the timeout of %sh:%sm)." % (timeout_hours, timeout_minutes)
                        msg += " Will force publication."
                    else:
                        msg += " (less than the timeout of %sh:%sm)." % (timeout_hours, timeout_minutes)
                        msg += " Not enough to force publication."
                    logger.info(wfnamemsg+msg)

        #logger.info(task[1])
        try:
            if self.force_publication:
                # - get info
                active_ = [{'key': [x['username'],
                                    x['user_group'],
                                    x['user_role'],
                                    x['taskname']],
                            'value': [x['destination'],
                                      x['source_lfn'],
                                      x['destination_lfn'],
                                      x['input_dataset'],
                                      x['dbs_url'],
                                      x['last_update']
                                     ]}
                           for x in task[1] if x['transfer_state'] == 3 and x['publication_state'] not in [2, 3, 5]]

                lfn_ready = []
                wf_jobs_endtime = []
                pnn, input_dataset, input_dbs_url = "", "", ""
                for active_file in active_:
                    job_end_time = active_file['value'][5]
                    if job_end_time and self.config.isOracle:
                        wf_jobs_endtime.append(int(job_end_time) - time.timezone)
                    elif job_end_time:
                        wf_jobs_endtime.append(int(time.mktime(time.strptime(str(job_end_time), '%Y-%m-%d %H:%M:%S'))) - time.timezone)
                    source_lfn = active_file['value'][1]
                    dest_lfn = active_file['value'][2]
                    self.lfn_map[dest_lfn] = source_lfn
                    if not pnn or not input_dataset or not input_dbs_url:
                        pnn = str(active_file['value'][0])
                        input_dataset = str(active_file['value'][3])
                        input_dbs_url = str(active_file['value'][4])
                    lfn_ready.append(dest_lfn)

                userDN = ''
                username = task[0][0]
                user_group = ""
                if task[0][1]:
                    user_group = task[0][1]
                user_role = ""
                if task[0][2]:
                    user_role = task[0][2]
                logger.debug("Trying to get DN %s %s %s" % (username, user_group, user_role))

                try:
                    userDN = getDNFromUserName(username, logger)
                except Exception as ex:
                    msg = "Error retrieving the user DN"
                    msg += str(ex)
                    msg += str(traceback.format_exc())
                    logger.error(msg)
                    return 1

                # Get metadata
                toPublish = []
                publDescFiles_list = self.getPublDescFiles(workflow, lfn_ready)
                for file_ in active_:
                    for _, doc in enumerate(publDescFiles_list):
                        #logger.info(type(doc))
                        #logger.info(doc)
                        if doc["lfn"] == file_["value"][2]:
                            doc["User"] = username
                            doc["Group"] = file_["key"][1]
                            doc["Role"] = file_["key"][2]
                            doc["UserDN"] = userDN
                            doc["Destination"] = file_["value"][0]
                            doc["SourceLFN"] = file_["value"][1]
                            toPublish.append(doc)
                with open("/tmp/"+workflow+'.json', 'w') as outfile:
                    json.dump(toPublish, outfile)
                logger.info(". publisher.sh %s" % (workflow))
                subprocess.call(["/bin/bash", "/data/user/MicroASO/microPublisher/python/publisher.sh", workflow])

        except:
            logger.exception("Exception!")


        return 0
Example #42
0
class PyCurlManager(unittest.TestCase):
    """Test pycurl_manager module"""

    def setUp(self):
        "initialization"
        self.mgr = RequestHandler()
        self.ckey = os.path.join(os.environ['HOME'], '.globus/userkey.pem')
        self.cert = os.path.join(os.environ['HOME'], '.globus/usercert.pem')

        self.cricheader = 'Date: Tue, 06 Nov 2018 14:50:29 GMT\r\nServer: Apache/2.4.6 (CentOS) OpenSSL/1.0.2k-fips mod_wsgi/3.4 Python/2.7.5 mod_gridsite/2.3.4\r\nVary: Cookie\r\nX-Frame-Options: SAMEORIGIN\r\nSet-Cookie: sessionid=bc1xu8zi5rbbsd5fgjuklb2tk2r3f6tw; expires=Sun, 11-Nov-2018 14:50:29 GMT; httponly; Max-Age=432000; Path=/\r\nContent-Length: 32631\r\nContent-Type: application/json\r\n\r\n'
        self.dbsheader = 'Date: Tue, 06 Nov 2018 14:39:07 GMT\r\nServer: Apache\r\nCMS-Server-Time: D=1503 t=1541515147806112\r\nTransfer-Encoding: chunked\r\nContent-Type: text/html\r\n\r\n'
        self.HTTPheader = 'Date: Tue, 06 Nov 2018 14:50:29 GMT\r\nServer: Apache/2.4.6 (CentOS) OpenSSL/1.0.2k-fips mod_wsgi/3.4 Python/2.7.5 mod_gridsite/2.3.4\r\nVary: Cookie\r\nX-Frame-Options: SAMEORIGIN\r\nSet-Cookie: GRIDHTTP_PASSCODE=2c6da9c96efa2ad0farhda; domain=cms-cric.cern.ch; path=/; secure\r\nContent-Length: 32631\r\nContent-Type: application/json\r\n\r\n'

    def testMulti(self):
        """
        Test fetch of several urls at once, one of the url relies on CERN SSO.
        """
        tfile = tempfile.NamedTemporaryFile()
        url1 = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader/help"
        url2 = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader/datatiers"
        url3 = "https://cms-gwmsmon.cern.ch/prodview/json/site_summary"
        cern_sso_cookie(url3, tfile.name, self.cert, self.ckey)
        cookie = {url3: tfile.name}
        urls = [url1, url2, url3]
        data = getdata(urls, self.ckey, self.cert, cookie=cookie)
        headers = 0
        for row in data:
            if '200 OK' in row['headers']:
                headers += 1
        self.assertTrue(headers, 3)

    def testSingle(self):
        """
        Test single call to CERN SSO url.
        """
        # test RequestHandler
        url = "https://cms-gwmsmon.cern.ch/prodview/json/site_summary"
        params = {}
        tfile = tempfile.NamedTemporaryFile()
        cern_sso_cookie(url, tfile.name, self.cert, self.ckey)
        cookie = {url: tfile.name}
        header, _ = self.mgr.request(url, params, cookie=cookie)
        self.assertTrue(header.status, 200)

    def testContinue(self):
        """
        Test HTTP exit code 100 - Continue
        """
        header = "HTTP/1.1 100 Continue\r\n" + self.dbsheader

        resp = ResponseHeader(header)
        self.assertIsNone(getattr(resp, "status", None))
        self.assertEqual(resp.reason, "")
        self.assertFalse(resp.fromcache)
        self.assertIn("CMS-Server-Time", resp.header)
        self.assertIn("Date", resp.header)
        self.assertEqual(resp.header['Content-Type'], 'text/html')
        self.assertEqual(resp.header['Server'], 'Apache')
        self.assertEqual(resp.header['Transfer-Encoding'], 'chunked')
        return

    def testOK(self):
        """
        Test HTTP exit code 200 - OK
        """
        header = "HTTP/1.1 200 OK\r\n" + self.dbsheader

        resp = ResponseHeader(header)
        self.assertEqual(resp.status, 200)
        self.assertEqual(resp.reason, "OK")
        self.assertFalse(resp.fromcache)
        return

    def testForbidden(self):
        """
        Test HTTP exit code 403 - Forbidden
        """
        header = "HTTP/1.1 403 Forbidden\r\n" + self.dbsheader

        resp = ResponseHeader(header)
        self.assertEqual(resp.status, 403)
        self.assertEqual(resp.reason, "Forbidden")
        self.assertFalse(resp.fromcache)
        return

    def testOKCRIC(self):
        """
        Test HTTP exit code 200 - OK for a CRIC response header
        """
        header = "HTTP/1.1 200 OK\r\n" + self.cricheader

        resp = ResponseHeader(header)
        self.assertEqual(resp.status, 200)
        self.assertEqual(resp.reason, "OK")
        self.assertFalse(resp.fromcache)
        self.assertIn("Content-Length", resp.header)
        self.assertIn("Date", resp.header)
        self.assertIn("Server", resp.header)
        self.assertIn("sessionid", resp.header['Set-Cookie'])
        self.assertEqual(resp.header['Content-Type'], 'application/json')
        self.assertEqual(resp.header['Vary'], 'Cookie')
        self.assertEqual(resp.header['X-Frame-Options'], 'SAMEORIGIN')
        return

    def testUnavailableCRICHTTP(self):
        """
        Test HTTP exit code 503 - Service Unavailable for a CRIC response header
        when it also contains a HTTP string in the Set-Cookie header section
        """
        header = "HTTP/1.1 503 Service Unavailable\r\n" + self.HTTPheader

        resp = ResponseHeader(header)
        self.assertEqual(resp.status, 503)
        self.assertEqual(resp.reason, "Service Unavailable")
        self.assertFalse(resp.fromcache)
        self.assertIn("Content-Length", resp.header)
        self.assertIn("Date", resp.header)
        self.assertIn("Server", resp.header)
        self.assertIn("GRIDHTTP_PASSCODE", resp.header['Set-Cookie'])
        self.assertEqual(resp.header['Content-Type'], 'application/json')
        self.assertEqual(resp.header['Vary'], 'Cookie')
        self.assertEqual(resp.header['X-Frame-Options'], 'SAMEORIGIN')
        return
Example #43
0
def getdata(url, params, headers=None):
    "Helper function to get data from the service"
    ckey, cert = getKeyCertFromEnv()
    mgr = RequestHandler()
    res = mgr.getdata(url, params=params, headers=headers, ckey=ckey, cert=cert)
    return json.loads(res)
Example #44
0
class PyCurlManager(unittest.TestCase):
    """Test pycurl_manager module"""
    def setUp(self):
        "initialization"
        self.mgr = RequestHandler()
        #self.ckey = os.path.join(os.environ['HOME'], '.globus/userkey.pem')
        #self.cert = os.path.join(os.environ['HOME'], '.globus/usercert.pem')
        self.ckey = getKeyCertFromEnv()[0]
        self.cert = getKeyCertFromEnv()[1]

        self.cricheader = 'Date: Tue, 06 Nov 2018 14:50:29 GMT\r\nServer: Apache/2.4.6 (CentOS) OpenSSL/1.0.2k-fips mod_wsgi/3.4 Python/2.7.5 mod_gridsite/2.3.4\r\nVary: Cookie\r\nX-Frame-Options: SAMEORIGIN\r\nSet-Cookie: sessionid=bc1xu8zi5rbbsd5fgjuklb2tk2r3f6tw; expires=Sun, 11-Nov-2018 14:50:29 GMT; httponly; Max-Age=432000; Path=/\r\nContent-Length: 32631\r\nContent-Type: application/json\r\n\r\n'
        self.dbsheader = 'Date: Tue, 06 Nov 2018 14:39:07 GMT\r\nServer: Apache\r\nCMS-Server-Time: D=1503 t=1541515147806112\r\nTransfer-Encoding: chunked\r\nContent-Type: text/html\r\n\r\n'
        self.HTTPheader = 'Date: Tue, 06 Nov 2018 14:50:29 GMT\r\nServer: Apache/2.4.6 (CentOS) OpenSSL/1.0.2k-fips mod_wsgi/3.4 Python/2.7.5 mod_gridsite/2.3.4\r\nVary: Cookie\r\nX-Frame-Options: SAMEORIGIN\r\nSet-Cookie: GRIDHTTP_PASSCODE=2c6da9c96efa2ad0farhda; domain=cms-cric.cern.ch; path=/; secure\r\nContent-Length: 32631\r\nContent-Type: application/json\r\n\r\n'

    def testMulti(self):
        """
        Test fetch of several urls at once, one of the url relies on CERN SSO.
        """
        tfile = tempfile.NamedTemporaryFile()
        url1 = "https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader/help"
        url2 = "https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader/datatiers"
        url3 = "https://cms-gwmsmon.cern.ch/prodview/json/site_summary"
        cern_sso_cookie(url3, tfile.name, self.cert, self.ckey)
        cookie = {url3: tfile.name}
        urls = [url1, url2, url3]
        data = getdata(urls, self.ckey, self.cert, cookie=cookie)
        headers = 0
        for row in data:
            if '200 OK' in row['headers']:
                headers += 1
        self.assertTrue(headers, 3)

    def testSingle(self):
        """
        Test single call to CERN SSO url.
        """
        # test RequestHandler
        url = "https://cms-gwmsmon.cern.ch/prodview/json/site_summary"
        params = {}
        headers = {"Cache-Control": "no-cache"}
        tfile = tempfile.NamedTemporaryFile()
        cern_sso_cookie(url, tfile.name, self.cert, self.ckey)
        cookie = {url: tfile.name}
        header, _ = self.mgr.request(url, params, headers, cookie=cookie)
        self.assertTrue(header.status, 200)

    def testContinue(self):
        """
        Test HTTP exit code 100 - Continue
        """
        header = "HTTP/1.1 100 Continue\r\n" + self.dbsheader

        resp = ResponseHeader(header)
        self.assertIsNone(getattr(resp, "status", None))
        self.assertEqual(resp.reason, "")
        self.assertFalse(resp.fromcache)
        self.assertIn("CMS-Server-Time", resp.header)
        self.assertIn("Date", resp.header)
        self.assertEqual(resp.header['Content-Type'], 'text/html')
        self.assertEqual(resp.header['Server'], 'Apache')
        self.assertEqual(resp.header['Transfer-Encoding'], 'chunked')
        return

    def testOK(self):
        """
        Test HTTP exit code 200 - OK
        """
        header = "HTTP/1.1 200 OK\r\n" + self.dbsheader

        resp = ResponseHeader(header)
        self.assertEqual(resp.status, 200)
        self.assertEqual(resp.reason, "OK")
        self.assertFalse(resp.fromcache)
        return

    def testForbidden(self):
        """
        Test HTTP exit code 403 - Forbidden
        """
        header = "HTTP/1.1 403 Forbidden\r\n" + self.dbsheader

        resp = ResponseHeader(header)
        self.assertEqual(resp.status, 403)
        self.assertEqual(resp.reason, "Forbidden")
        self.assertFalse(resp.fromcache)
        return

    def testOKCRIC(self):
        """
        Test HTTP exit code 200 - OK for a CRIC response header
        """
        header = "HTTP/1.1 200 OK\r\n" + self.cricheader

        resp = ResponseHeader(header)
        self.assertEqual(resp.status, 200)
        self.assertEqual(resp.reason, "OK")
        self.assertFalse(resp.fromcache)
        self.assertIn("Content-Length", resp.header)
        self.assertIn("Date", resp.header)
        self.assertIn("Server", resp.header)
        self.assertIn("sessionid", resp.header['Set-Cookie'])
        self.assertEqual(resp.header['Content-Type'], 'application/json')
        self.assertEqual(resp.header['Vary'], 'Cookie')
        self.assertEqual(resp.header['X-Frame-Options'], 'SAMEORIGIN')
        return

    def testUnavailableCRICHTTP(self):
        """
        Test HTTP exit code 503 - Service Unavailable for a CRIC response header
        when it also contains a HTTP string in the Set-Cookie header section
        """
        header = "HTTP/1.1 503 Service Unavailable\r\n" + self.HTTPheader

        resp = ResponseHeader(header)
        self.assertEqual(resp.status, 503)
        self.assertEqual(resp.reason, "Service Unavailable")
        self.assertFalse(resp.fromcache)
        self.assertIn("Content-Length", resp.header)
        self.assertIn("Date", resp.header)
        self.assertIn("Server", resp.header)
        self.assertIn("GRIDHTTP_PASSCODE", resp.header['Set-Cookie'])
        self.assertEqual(resp.header['Content-Type'], 'application/json')
        self.assertEqual(resp.header['Vary'], 'Cookie')
        self.assertEqual(resp.header['X-Frame-Options'], 'SAMEORIGIN')
        return

    def testHeadRequest(self):
        """
        Test a HEAD request.
        """
        params = {}
        headers = {}
        url = 'https://cmsweb.cern.ch/reqmgr2/data/info'
        res = self.mgr.getheader(url,
                                 params=params,
                                 headers=headers,
                                 ckey=self.ckey,
                                 cert=self.cert)
        self.assertEqual(res.getReason(), "OK")
        self.assertTrue(len(res.getHeader()) > 10)
        # Kubernetes cluster responds with a different Server header
        serverHeader = res.getHeaderKey("Server")
        self.assertTrue(
            serverHeader.startswith("nginx/")
            or serverHeader.startswith("CherryPy/")
            or serverHeader.startswith("openresty/"))
Example #45
0
class Requests(dict):
    """
    Generic class for sending different types of HTTP Request to a given URL
    """

    def __init__(self, url='http://localhost', idict=None):
        """
        url should really be host - TODO fix that when have sufficient code
        coverage and change _getURLOpener if needed
        """
        if not idict:
            idict = {}
        dict.__init__(self, idict)
        self.pycurl = idict.get('pycurl', None)
        self.capath = idict.get('capath', None)
        if self.pycurl:
            self.reqmgr = RequestHandler()

        # set up defaults
        self.setdefault("accept_type", 'text/html')
        self.setdefault("content_type", 'application/x-www-form-urlencoded')
        self.additionalHeaders = {}

        # check for basic auth early, as if found this changes the url
        urlComponent = sanitizeURL(url)
        if urlComponent['username'] is not None:
            self.addBasicAuth( \
                urlComponent['username'], urlComponent['password'])
            url = urlComponent['url']  # remove user, password from url

        self.setdefault("host", url)

        # then update with the incoming dict
        self.update(idict)

        self['endpoint_components'] = urlparse.urlparse(self['host'])

        # If cachepath = None disable caching
        if 'cachepath' in idict and idict['cachepath'] is None:
            self["req_cache_path"] = None
        else:
            cache_dir = (self.cachePath(idict.get('cachepath'), \
                                        idict.get('service_name')))
            self["cachepath"] = cache_dir
            self["req_cache_path"] = os.path.join(cache_dir, '.cache')
        self.setdefault("timeout", 300)
        self.setdefault("logger", logging)

        check_server_url(self['host'])

    def get(self, uri=None, data={}, incoming_headers={},
            encode=True, decode=True, contentType=None):
        """
        GET some data
        """
        return self.makeRequest(uri, data, 'GET', incoming_headers,
                                encode, decode, contentType)

    def post(self, uri=None, data={}, incoming_headers={},
             encode=True, decode=True, contentType=None):
        """
        POST some data
        """
        return self.makeRequest(uri, data, 'POST', incoming_headers,
                                encode, decode, contentType)

    def put(self, uri=None, data={}, incoming_headers={},
            encode=True, decode=True, contentType=None):
        """
        PUT some data
        """
        return self.makeRequest(uri, data, 'PUT', incoming_headers,
                                encode, decode, contentType)

    def delete(self, uri=None, data={}, incoming_headers={},
               encode=True, decode=True, contentType=None):
        """
        DELETE some data
        """
        return self.makeRequest(uri, data, 'DELETE', incoming_headers,
                                encode, decode, contentType)

    def makeRequest(self, uri=None, data={}, verb='GET', incoming_headers={},
                    encoder=True, decoder=True, contentType=None):
        """
        Wrapper around request helper functions.
        """
        if self.pycurl:
            result = self.makeRequest_pycurl(uri, data, verb, incoming_headers,
                                             encoder, decoder, contentType)
        else:
            result = self.makeRequest_httplib(uri, data, verb, incoming_headers,
                                              encoder, decoder, contentType)
        return result

    def makeRequest_pycurl(self, uri=None, params={}, verb='GET',
                           incoming_headers={}, encoder=True, decoder=True, contentType=None):
        """
        Make HTTP(s) request via pycurl library. Stay complaint with
        makeRequest_httplib method.
        """
        ckey, cert = self.getKeyCert()
        capath = self.getCAPath()
        if not contentType:
            contentType = self['content_type']
        headers = {"Content-type": contentType,
                   "User-agent": "WMCore.Services.Requests/v001",
                   "Accept": self['accept_type']}
        for key in self.additionalHeaders.keys():
            headers[key] = self.additionalHeaders[key]
        # And now overwrite any headers that have been passed into the call:
        headers.update(incoming_headers)
        url = self['host'] + uri
        response, data = self.reqmgr.request(url, params, headers, \
                                             verb=verb, ckey=ckey, cert=cert, capath=capath, decode=decoder)
        return data, response.status, response.reason, response.fromcache

    def makeRequest_httplib(self, uri=None, data={}, verb='GET',
                            incoming_headers={}, encoder=True, decoder=True, contentType=None):
        """
        Make a request to the remote database. for a give URI. The type of
        request will determine the action take by the server (be careful with
        DELETE!). Data should be a dictionary of {dataname: datavalue}.

        Returns a tuple of the data from the server, decoded using the
        appropriate method the response status and the response reason, to be
        used in error handling.

        You can override the method to encode/decode your data by passing in an
        encoding/decoding function to this method. Your encoded data must end up
        as a string.

        """
        # TODO: User agent should be:
        # $client/$client_version (CMS)
        # $http_lib/$http_lib_version $os/$os_version ($arch)
        if not contentType:
            contentType = self['content_type']
        headers = {"Content-type": contentType,
                   "User-agent": "WMCore.Services.Requests/v001",
                   "Accept": self['accept_type']}
        encoded_data = ''

        for key in self.additionalHeaders.keys():
            headers[key] = self.additionalHeaders[key]

        # And now overwrite any headers that have been passed into the call:
        # WARNING: doesn't work with deplate so only accept gzip
        incoming_headers["accept-encoding"] = "gzip,identity"
        headers.update(incoming_headers)

        # httpib2 requires absolute url
        uri = self['host'] + uri

        # If you're posting an attachment, the data might not be a dict
        #   please test against ConfigCache_t if you're unsure.
        # assert type(data) == type({}), \
        #        "makeRequest input data must be a dict (key/value pairs)"

        if verb != 'GET' and data:
            if isinstance(encoder, (types.MethodType, types.FunctionType)):
                encoded_data = encoder(data)
            elif encoder == False:
                # Don't encode the data more than we have to
                #  we don't want to URL encode the data blindly,
                #  that breaks POSTing attachments... ConfigCache_t
                # encoded_data = urllib.urlencode(data)
                #  -- Andrew Melo 25/7/09
                encoded_data = data
            else:
                # Either the encoder is set to True or it's junk, so use
                # self.encode
                encoded_data = self.encode(data)
            headers["Content-length"] = len(encoded_data)
        elif verb == 'GET' and data:
            # encode the data as a get string
            uri = "%s?%s" % (uri, urllib.urlencode(data, doseq=True))

        headers["Content-length"] = str(len(encoded_data))

        assert isinstance(encoded_data, str), \
            "Data in makeRequest is %s and not encoded to a string" % type(encoded_data)

        # httplib2 will allow sockets to close on remote end without retrying
        # try to send request - if this fails try again - should then succeed
        try:
            conn = self._getURLOpener()
            response, result = conn.request(uri, method=verb,
                                                    body=encoded_data, headers=headers)
            if response.status == 408:  # timeout can indicate a socket error
                raise socket.error
        except (socket.error, AttributeError):
            self['logger'].warn("Http request failed, retrying once again..")
            # AttributeError implies initial connection error - need to close
            # & retry. httplib2 doesn't clear httplib state before next request
            # if this is threaded this may spoil things
            # only have one endpoint so don't need to determine which to shut
            for con in conn.connections.values():
                con.close()
            conn = self._getURLOpener()
            # ... try again... if this fails propagate error to client
            try:
                response, result = conn.request(uri, method=verb,
                                                        body=encoded_data, headers=headers)
            except AttributeError:
                msg = traceback.format_exc()
                # socket/httplib really screwed up - nuclear option
                conn.connections = {}
                raise socket.error('Error contacting: %s: %s' % (self.getDomainName(), msg))
        if response.status >= 400:
            e = HTTPException()
            setattr(e, 'req_data', encoded_data)
            setattr(e, 'req_headers', headers)
            setattr(e, 'url', uri)
            setattr(e, 'result', result)
            setattr(e, 'status', response.status)
            setattr(e, 'reason', response.reason)
            setattr(e, 'headers', response)
            raise e

        if isinstance(decoder, (types.MethodType, types.FunctionType)):
            result = decoder(result)
        elif decoder != False:
            result = self.decode(result)
        # TODO: maybe just return result and response...
        return result, response.status, response.reason, response.fromcache

    def encode(self, data):
        """
        encode data into some appropriate format, for now make it a string...
        """
        return urllib.urlencode(data, doseq=1)

    def decode(self, data):
        """
        decode data to some appropriate format, for now make it a string...
        """
        return data.__str__()

    def cachePath(self, given_path, service_name):
        """Return cache location"""
        if not service_name:
            service_name = 'REQUESTS'
        top = self.cacheTopPath(given_path, service_name)

        # deal with multiple Services that have the same service running and
        # with multiple users for a given Service
        if self.getUserName() is None:
            cachepath = os.path.join(top, self['endpoint_components'].netloc)
        else:
            cachepath = os.path.join(top, '%s-%s' % (self.getUserName(), self.getDomainName()))

        try:
            # only we should be able to write to this dir
            os.makedirs(cachepath, stat.S_IRWXU)
        except OSError:
            if not os.path.isdir(cachepath):
                raise
            Permissions.owner_readwriteexec(cachepath)

        return cachepath

    def cacheTopPath(self, given_path, service_name):
        """Where to cache results?

        Logic:
          o If passed in take that
          o Is the environment variable "SERVICE_NAME"_CACHE_DIR defined?
          o Is WMCORE_CACHE_DIR set
          o Generate a temporary directory
          """
        if given_path:
            return given_path
        user = str(os.getuid())
        # append user id so users don't clobber each other
        lastbit = os.path.join('.wmcore_cache_%s' % user, service_name.lower())
        for var in ('%s_CACHE_DIR' % service_name.upper(),
                    'WMCORE_CACHE_DIR'):
            if os.environ.get(var):
                firstbit = os.environ[var]
                break
        else:
            idir = tempfile.mkdtemp(prefix='.wmcore_cache_')
            self['deleteCacheOnExit'] = TempDirectory(idir)
            return idir

        return os.path.join(firstbit, lastbit)

    def getDomainName(self):
        """Parse netloc info to get hostname"""
        return self['endpoint_components'].hostname

    def getUserName(self):
        """Parse netloc to get user"""
        return self['endpoint_components'].username

    def _getURLOpener(self):
        """
        method getting a secure (HTTPS) connection
        """
        import httplib2
        key, cert = None, None
        if self['endpoint_components'].scheme == 'https':
            # only add certs to https requests
            # if we have a key/cert add to request,
            # if not proceed as not all https connections require them
            try:
                key, cert = self.getKeyCert()
            except Exception as ex:
                msg = 'No certificate or key found, authentication may fail'
                self['logger'].info(msg)
                self['logger'].debug(str(ex))

        try:
            # disable validation as we don't have a single PEM with all ca's
            http = httplib2.Http(self['req_cache_path'], self['timeout'],
                                 disable_ssl_certificate_validation=True)
        except TypeError:
            # old httplib2 versions disable validation by default
            http = httplib2.Http(self['req_cache_path'], self['timeout'])

        # Domain must be just a hostname and port. self[host] is a URL currently
        if key or cert:
            http.add_certificate(key=key, cert=cert, domain='')
        return http

    def addBasicAuth(self, username, password):
        """Add basic auth headers to request"""
        auth_string = "Basic %s" % base64.encodestring('%s:%s' % (
            username, password)).strip()
        self.additionalHeaders["Authorization"] = auth_string

    def getKeyCert(self):
        """
       _getKeyCert_

       Get the user credentials if they exist, otherwise throw an exception.
       This code was modified from DBSAPI/dbsHttpService.py
        """
        cert = None
        key = None
        # Zeroth case is if the class has over ridden the key/cert and has it
        # stored in self
        if 'cert' in self and 'key' in self and self['cert'] and self['key']:
            key = self['key']
            cert = self['cert']

        # Now we're trying to guess what the right cert/key combo is...
        # First preference to HOST Certificate, This is how it set in Tier0
        elif 'X509_HOST_CERT' in os.environ:
            cert = os.environ['X509_HOST_CERT']
            key = os.environ['X509_HOST_KEY']
        # Second preference to User Proxy, very common
        elif 'X509_USER_PROXY' in os.environ and os.path.exists(os.environ['X509_USER_PROXY']):
            cert = os.environ['X509_USER_PROXY']
            key = cert
        # Third preference to User Cert/Proxy combinition
        elif 'X509_USER_CERT' in os.environ:
            cert = os.environ['X509_USER_CERT']
            key = os.environ['X509_USER_KEY']
        # TODO: only in linux, unix case, add other os case
        # look for proxy at default location /tmp/x509up_u$uid
        elif os.path.exists('/tmp/x509up_u' + str(os.getuid())):
            cert = '/tmp/x509up_u' + str(os.getuid())
            key = cert

        # if interactive we can use an encrypted certificate
        elif sys.stdin.isatty():
            if os.path.exists(os.environ['HOME'] + '/.globus/usercert.pem'):
                cert = os.environ['HOME'] + '/.globus/usercert.pem'
                if os.path.exists(os.environ['HOME'] + '/.globus/userkey.pem'):
                    key = os.environ['HOME'] + '/.globus/userkey.pem'
                else:
                    key = cert

        # Set but not found
        if key and cert:
            if not os.path.exists(cert) or not os.path.exists(key):
                raise WMException('Request requires a host certificate and key',
                                  "WMCORE-11")

        # All looks OK, still doesn't guarantee proxy's validity etc.
        return key, cert

    def getCAPath(self):
        """
        _getCAPath_

        Return the path of the CA certificates. The check is loose in the pycurl_manager:
        is capath == None then the server identity is not verified. To enable this check
        you need to set either the X509_CERT_DIR variable or the cacert key of the request.
        """
        cacert = None
        if 'capath' in self:
            cacert = self['capath']
        elif "X509_CERT_DIR" in os.environ:
            cacert = os.environ["X509_CERT_DIR"]
        return cacert

    def uploadFile(self, fileName, url, fieldName='file1', params=[], verb='POST'):
        """
        Upload a file with curl streaming it directly from disk
        """
        ckey, cert = self.getKeyCert()
        capath = self.getCAPath()
        import pycurl
        c = pycurl.Curl()
        if verb == 'POST':
            c.setopt(c.POST, 1)
        elif verb == 'PUT':
            c.setopt(pycurl.CUSTOMREQUEST, 'PUT')
        else:
            raise HTTPException("Verb %s not sopported for upload." % verb)
        c.setopt(c.URL, url)
        fullParams = [(fieldName, (c.FORM_FILE, fileName))]
        fullParams.extend(params)
        c.setopt(c.HTTPPOST, fullParams)
        bbuf = StringIO.StringIO()
        hbuf = StringIO.StringIO()
        c.setopt(pycurl.WRITEFUNCTION, bbuf.write)
        c.setopt(pycurl.HEADERFUNCTION, hbuf.write)
        if capath:
            c.setopt(pycurl.CAPATH, capath)
            c.setopt(pycurl.SSL_VERIFYPEER, True)
        else:
            c.setopt(pycurl.SSL_VERIFYPEER, False)
        if ckey:
            c.setopt(pycurl.SSLKEY, ckey)
        if cert:
            c.setopt(pycurl.SSLCERT, cert)
        c.perform()
        hres = hbuf.getvalue()
        bres = bbuf.getvalue()
        rh = ResponseHeader(hres)
        c.close()
        if rh.status < 200 or rh.status >= 300:
            exc = HTTPException(bres)
            setattr(exc, 'req_data', fullParams)
            setattr(exc, 'url', url)
            setattr(exc, 'result', bres)
            setattr(exc, 'status', rh.status)
            setattr(exc, 'reason', rh.reason)
            setattr(exc, 'headers', rh.header)
            raise exc

        return bres

    def downloadFile(self, fileName, url):
        """
        Download a file with curl streaming it directly to disk
        """
        ckey, cert = self.getKeyCert()
        capath = self.getCAPath()
        import pycurl
        from WMCore.Services.pycurl_manager import ResponseHeader

        hbuf = StringIO.StringIO()

        with open(fileName, "wb") as fp:
            curl = pycurl.Curl()
            curl.setopt(pycurl.URL, url)
            curl.setopt(pycurl.WRITEDATA, fp)
            curl.setopt(pycurl.HEADERFUNCTION, hbuf.write)
            if capath:
                curl.setopt(pycurl.CAPATH, capath)
                curl.setopt(pycurl.SSL_VERIFYPEER, True)
            else:
                curl.setopt(pycurl.SSL_VERIFYPEER, False)
            if ckey:
                curl.setopt(pycurl.SSLKEY, ckey)
            if cert:
                curl.setopt(pycurl.SSLCERT, cert)
            curl.setopt(pycurl.FOLLOWLOCATION, 1)
            curl.perform()
            curl.close()

            header = ResponseHeader(hbuf.getvalue())
            if header.status < 200 or header.status >= 300:
                raise RuntimeError('Reading %s failed with code %s' % (url, header.status))
        return fileName, header
Example #46
0
class Requests(dict):
    """
    Generic class for sending different types of HTTP Request to a given URL
    """
    def __init__(self, url='http://localhost', idict=None):
        """
        url should really be host - TODO fix that when have sufficient code
        coverage and change _getURLOpener if needed
        """
        if not idict:
            idict = {}
        dict.__init__(self, idict)
        self.pycurl = idict.get('pycurl', None)
        self.capath = idict.get('capath', None)
        if self.pycurl:
            self.reqmgr = RequestHandler()

        # set up defaults
        self.setdefault("accept_type", 'text/html')
        self.setdefault("content_type", 'application/x-www-form-urlencoded')
        self.additionalHeaders = {}

        # check for basic auth early, as if found this changes the url
        urlComponent = sanitizeURL(url)
        if urlComponent['username'] is not None:
            self.addBasicAuth(urlComponent['username'],
                              urlComponent['password'])
            url = urlComponent['url']  # remove user, password from url

        self.setdefault("host", url)

        # then update with the incoming dict
        self.update(idict)

        self['endpoint_components'] = urlparse.urlparse(self['host'])

        # If cachepath = None disable caching
        if 'cachepath' in idict and idict['cachepath'] is None:
            self["req_cache_path"] = None
        else:
            cache_dir = (self.cachePath(idict.get('cachepath'),
                                        idict.get('service_name')))
            self["cachepath"] = cache_dir
            self["req_cache_path"] = os.path.join(cache_dir, '.cache')
        self.setdefault("cert", None)
        self.setdefault("key", None)
        self.setdefault('capath', None)
        self.setdefault("timeout", 300)
        self.setdefault("logger", logging)

        check_server_url(self['host'])

    def get(self,
            uri=None,
            data={},
            incoming_headers={},
            encode=True,
            decode=True,
            contentType=None):
        """
        GET some data
        """
        return self.makeRequest(uri, data, 'GET', incoming_headers, encode,
                                decode, contentType)

    def post(self,
             uri=None,
             data={},
             incoming_headers={},
             encode=True,
             decode=True,
             contentType=None):
        """
        POST some data
        """
        return self.makeRequest(uri, data, 'POST', incoming_headers, encode,
                                decode, contentType)

    def put(self,
            uri=None,
            data={},
            incoming_headers={},
            encode=True,
            decode=True,
            contentType=None):
        """
        PUT some data
        """
        return self.makeRequest(uri, data, 'PUT', incoming_headers, encode,
                                decode, contentType)

    def delete(self,
               uri=None,
               data={},
               incoming_headers={},
               encode=True,
               decode=True,
               contentType=None):
        """
        DELETE some data
        """
        return self.makeRequest(uri, data, 'DELETE', incoming_headers, encode,
                                decode, contentType)

    def makeRequest(self,
                    uri=None,
                    data={},
                    verb='GET',
                    incoming_headers={},
                    encoder=True,
                    decoder=True,
                    contentType=None):
        """
        Wrapper around request helper functions.
        """
        if self.pycurl:
            result = self.makeRequest_pycurl(uri, data, verb, incoming_headers,
                                             encoder, decoder, contentType)
        else:
            result = self.makeRequest_httplib(uri, data, verb,
                                              incoming_headers, encoder,
                                              decoder, contentType)
        return result

    def makeRequest_pycurl(self,
                           uri=None,
                           params={},
                           verb='GET',
                           incoming_headers={},
                           encoder=True,
                           decoder=True,
                           contentType=None):
        """
        Make HTTP(s) request via pycurl library. Stay complaint with
        makeRequest_httplib method.
        """
        ckey, cert = self.getKeyCert()
        capath = self.getCAPath()
        if not contentType:
            contentType = self['content_type']
        headers = {
            "Content-type": contentType,
            "User-agent": "WMCore.Services.Requests/v001",
            "Accept": self['accept_type']
        }
        for key in self.additionalHeaders.keys():
            headers[key] = self.additionalHeaders[key]
        # And now overwrite any headers that have been passed into the call:
        headers.update(incoming_headers)
        url = self['host'] + uri
        response, data = self.reqmgr.request(url,
                                             params,
                                             headers,
                                             verb=verb,
                                             ckey=ckey,
                                             cert=cert,
                                             capath=capath,
                                             decode=decoder)
        return data, response.status, response.reason, response.fromcache

    def makeRequest_httplib(self,
                            uri=None,
                            data={},
                            verb='GET',
                            incoming_headers={},
                            encoder=True,
                            decoder=True,
                            contentType=None):
        """
        Make a request to the remote database. for a give URI. The type of
        request will determine the action take by the server (be careful with
        DELETE!). Data should be a dictionary of {dataname: datavalue}.

        Returns a tuple of the data from the server, decoded using the
        appropriate method the response status and the response reason, to be
        used in error handling.

        You can override the method to encode/decode your data by passing in an
        encoding/decoding function to this method. Your encoded data must end up
        as a string.

        """
        # TODO: User agent should be:
        # $client/$client_version (CMS)
        # $http_lib/$http_lib_version $os/$os_version ($arch)
        if not contentType:
            contentType = self['content_type']
        headers = {
            "Content-type": contentType,
            "User-agent": "WMCore.Services.Requests/v001",
            "Accept": self['accept_type']
        }
        encoded_data = ''

        for key in self.additionalHeaders.keys():
            headers[key] = self.additionalHeaders[key]

        # And now overwrite any headers that have been passed into the call:
        # WARNING: doesn't work with deplate so only accept gzip
        incoming_headers["accept-encoding"] = "gzip,identity"
        headers.update(incoming_headers)

        # httpib2 requires absolute url
        uri = self['host'] + uri

        # If you're posting an attachment, the data might not be a dict
        #   please test against ConfigCache_t if you're unsure.
        # assert type(data) == type({}), \
        #        "makeRequest input data must be a dict (key/value pairs)"

        if verb != 'GET' and data:
            if isinstance(encoder, (types.MethodType, types.FunctionType)):
                encoded_data = encoder(data)
            elif encoder == False:
                # Don't encode the data more than we have to
                #  we don't want to URL encode the data blindly,
                #  that breaks POSTing attachments... ConfigCache_t
                # encoded_data = urllib.urlencode(data)
                #  -- Andrew Melo 25/7/09
                encoded_data = data
            else:
                # Either the encoder is set to True or it's junk, so use
                # self.encode
                encoded_data = self.encode(data)
            headers["Content-length"] = len(encoded_data)
        elif verb == 'GET' and data:
            # encode the data as a get string
            uri = "%s?%s" % (uri, urllib.urlencode(data, doseq=True))

        headers["Content-length"] = str(len(encoded_data))

        # PY3 needed for compatibility because str under futurize is not a string. Can be just str in Py3 only
        # PY3 Don't let futurize change this
        assert isinstance(encoded_data, (str, basestring)), \
            "Data in makeRequest is %s and not encoded to a string" % type(encoded_data)

        # httplib2 will allow sockets to close on remote end without retrying
        # try to send request - if this fails try again - should then succeed
        try:
            conn = self._getURLOpener()
            response, result = conn.request(uri,
                                            method=verb,
                                            body=encoded_data,
                                            headers=headers)
            if response.status == 408:  # timeout can indicate a socket error
                raise socket.error
        except ServerNotFoundError as ex:
            # DNS cannot resolve this domain name, let's call it 'Service Unavailable'
            e = HTTPException()
            setattr(e, 'url', uri)
            setattr(e, 'status', 503)
            setattr(e, 'reason', 'Service Unavailable')
            setattr(e, 'result', str(ex))
            raise e
        except (socket.error, AttributeError):
            self['logger'].warn("Http request failed, retrying once again..")
            # AttributeError implies initial connection error - need to close
            # & retry. httplib2 doesn't clear httplib state before next request
            # if this is threaded this may spoil things
            # only have one endpoint so don't need to determine which to shut
            for con in conn.connections.values():
                con.close()
            conn = self._getURLOpener()
            # ... try again... if this fails propagate error to client
            try:
                response, result = conn.request(uri,
                                                method=verb,
                                                body=encoded_data,
                                                headers=headers)
            except AttributeError:
                msg = traceback.format_exc()
                # socket/httplib really screwed up - nuclear option
                conn.connections = {}
                raise socket.error('Error contacting: %s: %s' %
                                   (self.getDomainName(), msg))
        if response.status >= 400:
            e = HTTPException()
            setattr(e, 'req_data', encoded_data)
            setattr(e, 'req_headers', headers)
            setattr(e, 'url', uri)
            setattr(e, 'result', result)
            setattr(e, 'status', response.status)
            setattr(e, 'reason', response.reason)
            setattr(e, 'headers', response)
            raise e

        if isinstance(decoder, (types.MethodType, types.FunctionType)):
            result = decoder(result)
        elif decoder != False:
            result = self.decode(result)
        # TODO: maybe just return result and response...
        return result, response.status, response.reason, response.fromcache

    def encode(self, data):
        """
        encode data into some appropriate format, for now make it a string...
        """
        return urllib.urlencode(data, doseq=1)

    def decode(self, data):
        """
        decode data to some appropriate format, for now make it a string...
        """
        return data.__str__()

    def cachePath(self, given_path, service_name):
        """Return cache location"""
        if not service_name:
            service_name = 'REQUESTS'
        top = self.cacheTopPath(given_path, service_name)

        # deal with multiple Services that have the same service running and
        # with multiple users for a given Service
        if self.getUserName() is None:
            cachepath = os.path.join(top, self['endpoint_components'].netloc)
        else:
            cachepath = os.path.join(
                top, '%s-%s' % (self.getUserName(), self.getDomainName()))

        try:
            # only we should be able to write to this dir
            os.makedirs(cachepath, stat.S_IRWXU)
        except OSError:
            if not os.path.isdir(cachepath):
                raise
            Permissions.owner_readwriteexec(cachepath)

        return cachepath

    def cacheTopPath(self, given_path, service_name):
        """Where to cache results?

        Logic:
          o If passed in take that
          o Is the environment variable "SERVICE_NAME"_CACHE_DIR defined?
          o Is WMCORE_CACHE_DIR set
          o Generate a temporary directory
          """
        if given_path:
            return given_path
        user = str(os.getuid())
        # append user id so users don't clobber each other
        lastbit = os.path.join('.wmcore_cache_%s' % user, service_name.lower())
        for var in ('%s_CACHE_DIR' % service_name.upper(), 'WMCORE_CACHE_DIR'):
            if os.environ.get(var):
                firstbit = os.environ[var]
                break
        else:
            idir = tempfile.mkdtemp(prefix='.wmcore_cache_')
            self['deleteCacheOnExit'] = TempDirectory(idir)
            return idir

        return os.path.join(firstbit, lastbit)

    def getDomainName(self):
        """Parse netloc info to get hostname"""
        return self['endpoint_components'].hostname

    def getUserName(self):
        """Parse netloc to get user"""
        return self['endpoint_components'].username

    def _getURLOpener(self):
        """
        method getting a secure (HTTPS) connection
        """
        import httplib2
        key, cert = None, None
        if self['endpoint_components'].scheme == 'https':
            # only add certs to https requests
            # if we have a key/cert add to request,
            # if not proceed as not all https connections require them
            try:
                key, cert = self.getKeyCert()
            except Exception as ex:
                msg = 'No certificate or key found, authentication may fail'
                self['logger'].info(msg)
                self['logger'].debug(str(ex))

        try:
            # disable validation as we don't have a single PEM with all ca's
            http = httplib2.Http(self['req_cache_path'],
                                 self['timeout'],
                                 disable_ssl_certificate_validation=True)
        except TypeError:
            # old httplib2 versions disable validation by default
            http = httplib2.Http(self['req_cache_path'], self['timeout'])

        # Domain must be just a hostname and port. self[host] is a URL currently
        if key or cert:
            http.add_certificate(key=key, cert=cert, domain='')
        return http

    def addBasicAuth(self, username, password):
        """Add basic auth headers to request"""
        auth_string = "Basic %s" % base64.encodestring(
            '%s:%s' % (username, password)).strip()
        self.additionalHeaders["Authorization"] = auth_string

    def getKeyCert(self):
        """
       _getKeyCert_

       Get the user credentials if they exist, otherwise throw an exception.
       This code was modified from DBSAPI/dbsHttpService.py
        """

        # Zeroth case is if the class has over ridden the key/cert and has it
        # stored in self
        if self['cert'] and self['key']:
            key = self['key']
            cert = self['cert']
        else:
            key, cert = getKeyCertFromEnv()

        # Set but not found
        if key is None or cert is None:
            raise WMException('Request requires a host certificate and key',
                              "WMCORE-11")

        # All looks OK, still doesn't guarantee proxy's validity etc.
        return key, cert

    def getCAPath(self):
        """
        _getCAPath_

        Return the path of the CA certificates. The check is loose in the pycurl_manager:
        is capath == None then the server identity is not verified. To enable this check
        you need to set either the X509_CERT_DIR variable or the cacert key of the request.
        """
        capath = self['capath']
        if not capath:
            capath = getCAPathFromEnv()
        return capath

    def uploadFile(self,
                   fileName,
                   url,
                   fieldName='file1',
                   params=[],
                   verb='POST'):
        """
        Upload a file with curl streaming it directly from disk
        """
        ckey, cert = self.getKeyCert()
        capath = self.getCAPath()
        import pycurl
        c = pycurl.Curl()
        if verb == 'POST':
            c.setopt(c.POST, 1)
        elif verb == 'PUT':
            c.setopt(pycurl.CUSTOMREQUEST, 'PUT')
        else:
            raise HTTPException("Verb %s not sopported for upload." % verb)
        c.setopt(c.URL, url)
        fullParams = [(fieldName, (c.FORM_FILE, fileName))]
        fullParams.extend(params)
        c.setopt(c.HTTPPOST, fullParams)
        bbuf = StringIO.StringIO()
        hbuf = StringIO.StringIO()
        c.setopt(pycurl.WRITEFUNCTION, bbuf.write)
        c.setopt(pycurl.HEADERFUNCTION, hbuf.write)
        if capath:
            c.setopt(pycurl.CAPATH, capath)
            c.setopt(pycurl.SSL_VERIFYPEER, True)
        else:
            c.setopt(pycurl.SSL_VERIFYPEER, False)
        if ckey:
            c.setopt(pycurl.SSLKEY, ckey)
        if cert:
            c.setopt(pycurl.SSLCERT, cert)
        c.perform()
        hres = hbuf.getvalue()
        bres = bbuf.getvalue()
        rh = ResponseHeader(hres)
        c.close()
        if rh.status < 200 or rh.status >= 300:
            exc = HTTPException(bres)
            setattr(exc, 'req_data', fullParams)
            setattr(exc, 'url', url)
            setattr(exc, 'result', bres)
            setattr(exc, 'status', rh.status)
            setattr(exc, 'reason', rh.reason)
            setattr(exc, 'headers', rh.header)
            raise exc

        return bres

    def downloadFile(self, fileName, url):
        """
        Download a file with curl streaming it directly to disk
        """
        ckey, cert = self.getKeyCert()
        capath = self.getCAPath()
        import pycurl

        hbuf = StringIO.StringIO()

        with open(fileName, "wb") as fp:
            curl = pycurl.Curl()
            curl.setopt(pycurl.URL, url)
            curl.setopt(pycurl.WRITEDATA, fp)
            curl.setopt(pycurl.HEADERFUNCTION, hbuf.write)
            if capath:
                curl.setopt(pycurl.CAPATH, capath)
                curl.setopt(pycurl.SSL_VERIFYPEER, True)
            else:
                curl.setopt(pycurl.SSL_VERIFYPEER, False)
            if ckey:
                curl.setopt(pycurl.SSLKEY, ckey)
            if cert:
                curl.setopt(pycurl.SSLCERT, cert)
            curl.setopt(pycurl.FOLLOWLOCATION, 1)
            curl.perform()
            curl.close()

            header = ResponseHeader(hbuf.getvalue())
            if header.status < 200 or header.status >= 300:
                raise RuntimeError('Reading %s failed with code %s' %
                                   (url, header.status))
        return fileName, header
Example #47
0
class Requests(dict):
    """
    Generic class for sending different types of HTTP Request to a given URL
    """

    def __init__(self, url = 'http://localhost', idict=None):
        """
        url should really be host - TODO fix that when have sufficient code
        coverage and change _getURLOpener if needed
        """
        if  not idict:
            idict = {}
        dict.__init__(self, idict)
        self.pycurl = idict.get('pycurl', None)
        if self.pycurl:
            self.reqmgr = RequestHandler()

        #set up defaults
        self.setdefault("accept_type", 'text/html')
        self.setdefault("content_type", 'application/x-www-form-urlencoded')
        self.additionalHeaders = {}

        # check for basic auth early, as if found this changes the url
        urlComponent = sanitizeURL(url)
        if urlComponent['username'] is not None:
            self.addBasicAuth(\
                urlComponent['username'], urlComponent['password'])
            url = urlComponent['url'] # remove user, password from url

        self.setdefault("host", url)

        # then update with the incoming dict
        self.update(idict)

        self['endpoint_components'] = urlparse.urlparse(self['host'])

        # If cachepath = None disable caching
        if 'cachepath' in idict and idict['cachepath'] is None:
            self["req_cache_path"] = None
        else:
            cache_dir = (self.cachePath(idict.get('cachepath'), \
                        idict.get('service_name')))
            self["cachepath"] = cache_dir
            self["req_cache_path"] = os.path.join(cache_dir, '.cache')
        self.setdefault("timeout", 30)
        self.setdefault("logger", logging)

        check_server_url(self['host'])
        # and then get the URL opener
        self.setdefault("conn", self._getURLOpener())


    def get(self, uri=None, data={}, incoming_headers={},
               encode = True, decode=True, contentType=None):
        """
        GET some data
        """
        return self.makeRequest(uri, data, 'GET', incoming_headers,
                                encode, decode, contentType)

    def post(self, uri=None, data={}, incoming_headers={},
               encode = True, decode=True, contentType=None):
        """
        POST some data
        """
        return self.makeRequest(uri, data, 'POST', incoming_headers,
                                encode, decode, contentType)

    def put(self, uri=None, data={}, incoming_headers={},
               encode = True, decode=True, contentType=None):
        """
        PUT some data
        """
        return self.makeRequest(uri, data, 'PUT', incoming_headers,
                                encode, decode, contentType)

    def delete(self, uri=None, data={}, incoming_headers={},
               encode = True, decode=True, contentType=None):
        """
        DELETE some data
        """
        return self.makeRequest(uri, data, 'DELETE', incoming_headers,
                                encode, decode, contentType)

    def makeRequest(self, uri=None, data={}, verb='GET', incoming_headers={},
                     encoder=True, decoder=True, contentType=None):
        """
        Wrapper around request helper functions.
        """
        if  self.pycurl:
            result = self.makeRequest_pycurl(uri, data, verb, incoming_headers,
                         encoder, decoder, contentType)
        else:
            result = self.makeRequest_httplib(uri, data, verb, incoming_headers,
                         encoder, decoder, contentType)
        return result

    def makeRequest_pycurl(self, uri=None, params={}, verb='GET',
            incoming_headers={}, encoder=True, decoder=True, contentType=None):
        """
        Make HTTP(s) request via pycurl library. Stay complaint with
        makeRequest_httplib method.
        """
        ckey, cert = self.getKeyCert()
        capath = self.getCAPath()
        if  not contentType:
            contentType = self['content_type']
        headers = {"Content-type": contentType,
               "User-agent": "WMCore.Services.Requests/v001",
               "Accept": self['accept_type']}
        for key in self.additionalHeaders.keys():
            headers[key] = self.additionalHeaders[key]
        #And now overwrite any headers that have been passed into the call:
        headers.update(incoming_headers)
        url = self['host'] + uri
        response, data = self.reqmgr.request(url, params, headers, \
                    verb=verb, ckey=ckey, cert=cert, capath=capath, decode=decoder)
        return data, response.status, response.reason, response.fromcache

    def makeRequest_httplib(self, uri=None, data={}, verb='GET',
            incoming_headers={}, encoder=True, decoder=True, contentType=None):
        """
        Make a request to the remote database. for a give URI. The type of
        request will determine the action take by the server (be careful with
        DELETE!). Data should be a dictionary of {dataname: datavalue}.

        Returns a tuple of the data from the server, decoded using the
        appropriate method the response status and the response reason, to be
        used in error handling.

        You can override the method to encode/decode your data by passing in an
        encoding/decoding function to this method. Your encoded data must end up
        as a string.

        """
        #TODO: User agent should be:
        # $client/$client_version (CMS)
        # $http_lib/$http_lib_version $os/$os_version ($arch)
        if  not contentType:
            contentType = self['content_type']
        headers = {"Content-type": contentType,
               "User-agent": "WMCore.Services.Requests/v001",
               "Accept": self['accept_type']}
        encoded_data = ''

        for key in self.additionalHeaders.keys():
            headers[key] = self.additionalHeaders[key]

        #And now overwrite any headers that have been passed into the call:
        headers.update(incoming_headers)

        # httpib2 requires absolute url
        uri = self['host'] + uri

        # If you're posting an attachment, the data might not be a dict
        #   please test against ConfigCache_t if you're unsure.
        #assert type(data) == type({}), \
        #        "makeRequest input data must be a dict (key/value pairs)"

        # There must be a better way to do this...
        def f():
            """Dummy function"""
            pass

        if verb != 'GET' and data:
            if type(encoder) == type(self.get) or type(encoder) == type(f):
                encoded_data = encoder(data)
            elif encoder == False:
                # Don't encode the data more than we have to
                #  we don't want to URL encode the data blindly,
                #  that breaks POSTing attachments... ConfigCache_t
                #encoded_data = urllib.urlencode(data)
                #  -- Andrew Melo 25/7/09
                encoded_data = data
            else:
                # Either the encoder is set to True or it's junk, so use
                # self.encode
                encoded_data = self.encode(data)
            headers["Content-length"] = len(encoded_data)
        elif verb == 'GET' and data:
            #encode the data as a get string
            uri = "%s?%s" % (uri, urllib.urlencode(data, doseq=True))

        headers["Content-length"] = str(len(encoded_data))

        assert type(encoded_data) == type('string'), \
            "Data in makeRequest is %s and not encoded to a string" \
                % type(encoded_data)

        # httplib2 will allow sockets to close on remote end without retrying
        # try to send request - if this fails try again - should then succeed
        try:
            response, result = self['conn'].request(uri, method = verb,
                                    body = encoded_data, headers = headers)
            if response.status == 408: # timeout can indicate a socket error
                response, result = self['conn'].request(uri, method = verb,
                                    body = encoded_data, headers = headers)
        except (socket.error, AttributeError):
            # AttributeError implies initial connection error - need to close
            # & retry. httplib2 doesn't clear httplib state before next request
            # if this is threaded this may spoil things
            # only have one endpoint so don't need to determine which to shut
            [conn.close() for conn in self['conn'].connections.values()]
            self['conn'] = self._getURLOpener()
            # ... try again... if this fails propagate error to client
            try:
                response, result = self['conn'].request(uri, method = verb,
                                    body = encoded_data, headers = headers)
            except AttributeError:
                # socket/httplib really screwed up - nuclear option
                self['conn'].connections = {}
                raise socket.error, 'Error contacting: %s' \
                        % self.getDomainName()
        if response.status >= 400:
            e = HTTPException()
            setattr(e, 'req_data', encoded_data)
            setattr(e, 'req_headers', headers)
            setattr(e, 'url', uri)
            setattr(e, 'result', result)
            setattr(e, 'status', response.status)
            setattr(e, 'reason', response.reason)
            setattr(e, 'headers', response)
            raise e

        if type(decoder) == type(self.makeRequest) or type(decoder) == type(f):
            result = decoder(result)
        elif decoder != False:
            result = self.decode(result)
        #TODO: maybe just return result and response...
        return result, response.status, response.reason, response.fromcache

    def encode(self, data):
        """
        encode data into some appropriate format, for now make it a string...
        """
        return urllib.urlencode(data, doseq=1)

    def decode(self, data):
        """
        decode data to some appropriate format, for now make it a string...
        """
        return data.__str__()

    def cachePath(self, given_path, service_name):
        """Return cache location"""
        if not service_name:
            service_name = 'REQUESTS'
        top = self.cacheTopPath(given_path, service_name)

        # deal with multiple Services that have the same service running and
        # with multiple users for a given Service
        if self.getUserName() is None:
            cachepath = os.path.join(top, self['endpoint_components'].netloc)
        else:
            cachepath = os.path.join(top, '%s-%s' \
                % (self.getUserName(), self.getDomainName()))

        try:
            # only we should be able to write to this dir
            os.makedirs(cachepath, stat.S_IRWXU)
        except OSError:
            if not os.path.isdir(cachepath):
                raise
            Permissions.owner_readwriteexec(cachepath)

        return cachepath

    def cacheTopPath(self, given_path, service_name):
        """Where to cache results?

        Logic:
          o If passed in take that
          o Is the environment variable "SERVICE_NAME"_CACHE_DIR defined?
          o Is WMCORE_CACHE_DIR set
          o Generate a temporary directory
          """
        if given_path:
            return given_path
        user = str(os.getuid())
        # append user id so users don't clobber each other
        lastbit = os.path.join('.wmcore_cache_%s' % user, service_name.lower())
        for var in ('%s_CACHE_DIR' % service_name.upper(),
                    'WMCORE_CACHE_DIR'):
            if os.environ.get(var):
                firstbit = os.environ[var]
                break
        else:
            idir = tempfile.mkdtemp(prefix='.wmcore_cache_')
            # object to store temporary directory - cleaned up on destruction
            self['deleteCacheOnExit'] = TempDirectory(idir)
            return idir

        return os.path.join(firstbit, lastbit)

    def getDomainName(self):
        """Parse netloc info to get hostname"""
        return self['endpoint_components'].hostname

    def getUserName(self):
        """Parse netloc to get user"""
        return self['endpoint_components'].username

    def _getURLOpener(self):
        """
        method getting a secure (HTTPS) connection
        """
        key, cert = None, None
        if self['endpoint_components'].scheme == 'https':
            # only add certs to https requests
            # if we have a key/cert add to request,
            # if not proceed as not all https connections require them
            try:
                key, cert = self.getKeyCert()
            except Exception, ex:
                msg = 'No certificate or key found, authentication may fail'
                self['logger'].info(msg)
                self['logger'].debug(str(ex))

        try:
            # disable validation as we don't have a single PEM with all ca's
            http = httplib2.Http(self['req_cache_path'], self['timeout'],
                                 disable_ssl_certificate_validation = True)
        except TypeError:
            # old httplib2 versions disable validation by default
            http = httplib2.Http(self['req_cache_path'], self['timeout'])

        # Domain must be just a hostname and port. self[host] is a URL currently
        if key or cert:
            http.add_certificate(key=key, cert=cert, domain='')
        return http
 def setUp(self):
     "initialization"
     self.mgr = RequestHandler()
     self.ckey = os.path.join(os.environ['HOME'], '.globus/userkey.pem')
     self.cert = os.path.join(os.environ['HOME'], '.globus/usercert.pem')
Example #49
0
class MSRuleCleaner(MSCore):
    """
    MSRuleCleaner.py class provides the logic used to clean the Rucio
    block level data placement rules created by WMAgent.
    """

    def __init__(self, msConfig, logger=None):
        """
        Runs the basic setup and initialization for the MSRuleCleaner module
        :param msConfig: micro service configuration
        """
        super(MSRuleCleaner, self).__init__(msConfig, logger=logger)

        self.msConfig.setdefault("verbose", True)
        self.msConfig.setdefault("interval", 60)
        self.msConfig.setdefault("services", ['ruleCleaner'])
        self.msConfig.setdefault("rucioWmaAccount", "wma_test")
        self.msConfig.setdefault("rucioMStrAccount", "wmcore_transferor")
        self.msConfig.setdefault('enableRealMode', False)

        self.mode = "RealMode" if self.msConfig['enableRealMode'] else "DryRunMode"
        self.curlMgr = RequestHandler()
        self.targetStatusRegex = re.compile(r'.*archived')
        self.logDB = LogDB(self.msConfig["logDBUrl"],
                           self.msConfig["logDBReporter"],
                           logger=self.logger)
        self.wmstatsSvc = WMStatsServer(self.msConfig['wmstatsUrl'], logger=self.logger)

        # Building all the Pipelines:
        pName = 'plineMSTrCont'
        self.plineMSTrCont = Pipeline(name=pName,
                                      funcLine=[Functor(self.setPlineMarker, pName),
                                                Functor(self.setParentDatasets),
                                                Functor(self.getRucioRules, 'container', self.msConfig['rucioMStrAccount']),
                                                Functor(self.cleanRucioRules)])
        pName = 'plineMSTrBlock'
        self.plineMSTrBlock = Pipeline(name=pName,
                                       funcLine=[Functor(self.setPlineMarker, pName),
                                                 Functor(self.setParentDatasets),
                                                 Functor(self.getRucioRules, 'block', self.msConfig['rucioMStrAccount']),
                                                 Functor(self.cleanRucioRules)])
        pName = 'plineAgentCont'
        self.plineAgentCont = Pipeline(name=pName,
                                       funcLine=[Functor(self.setPlineMarker, pName),
                                                 Functor(self.getRucioRules, 'container', self.msConfig['rucioWmaAccount']),
                                                 Functor(self.cleanRucioRules)])
        pName = 'plineAgentBlock'
        self.plineAgentBlock = Pipeline(name=pName,
                                        funcLine=[Functor(self.setPlineMarker, pName),
                                                  Functor(self.getRucioRules, 'block', self.msConfig['rucioWmaAccount']),
                                                  Functor(self.cleanRucioRules)])
        pName = 'plineArchive'
        self.plineArchive = Pipeline(name=pName,
                                     funcLine=[Functor(self.setPlineMarker, pName),
                                               Functor(self.findTargetStatus),
                                               Functor(self.setClean),
                                               Functor(self.setArchivalDelayExpired),
                                               Functor(self.setLogDBClean),
                                               Functor(self.archive)])

        # Building the different set of plines we will need later:
        # NOTE: The following are all the functional pipelines which are supposed to include
        #       a cleanup function and report cleanup status in the MSRuleCleanerWflow object
        self.cleanuplines = [self.plineMSTrCont,
                             self.plineMSTrBlock,
                             self.plineAgentCont,
                             self.plineAgentBlock]
        # Building an auxiliary list of cleanup pipeline names only:
        self.cleanupPipeNames = [pline.name for pline in self.cleanuplines]

        # Building lists of pipelines related only to Agents or MStransferror
        self.agentlines = [self.plineAgentCont,
                           self.plineAgentBlock]
        self.mstrlines = [self.plineMSTrCont,
                          self.plineMSTrBlock]

        # Initialization of the 'cleaned' and 'archived' counters:
        self.wfCounters = {'cleaned': {},
                           'archived': {'normalArchived': 0,
                                        'forceArchived': 0}}
        self.globalLocks = set()

    def getGlobalLocks(self):
        """
        Fetches the list of 'globalLocks' from wmstats server and the list of
        'parentLocks' from request manager. Stores/updates the unified set in
        the 'globalLocks' instance variable. Returns the resultant unified set.
        :return: A union set of the 'globalLocks' and the 'parentLocks' lists
        """
        self.logger.info("Fetching globalLocks list from wmstats server.")
        try:
            globalLocks = set(self.wmstatsSvc.getGlobalLocks())
        except Exception as ex:
            msg = "Failed to refresh global locks list for the current polling cycle. Error: %s "
            msg += "Skipping this polling cycle."
            self.logger.error(msg, str(ex))
            raise ex
        self.logger.info("Fetching parentLocks list from reqmgr2 server.")
        try:
            parentLocks = set(self.reqmgr2.getParentLocks())
        except Exception as ex:
            msg = "Failed to refresh parent locks list for the current poling cycle. Error: %s "
            msg += "Skipping this polling cycle."
            self.logger.error(msg, str(ex))
            raise ex
        self.globalLocks = globalLocks | parentLocks

    def resetCounters(self):
        """
        A simple function for zeroing the cleaned and archived counters.
        """
        for pline in self.cleanuplines:
            self.wfCounters['cleaned'][pline.name] = 0
        self.wfCounters['archived']['normalArchived'] = 0
        self.wfCounters['archived']['forceArchived'] = 0

    def execute(self, reqStatus):
        """
        Executes the whole ruleCleaner logic
        :return: summary
        """
        # start threads in MSManager which should call this method
        summary = dict(RULECLEANER_REPORT)

        self.currThread = current_thread()
        self.currThreadIdent = self.currThread.name
        self.updateReportDict(summary, "thread_id", self.currThreadIdent)
        self.resetCounters()
        self.logger.info("MSRuleCleaner is running in mode: %s.", self.mode)

        # Build the list of workflows to work on:
        try:
            requestRecords = {}
            for status in reqStatus:
                requestRecords.update(self.getRequestRecords(status))
        except Exception as err:  # general error
            msg = "Unknown exception while fetching requests from ReqMgr2. Error: %s", str(err)
            self.logger.exception(msg)
            self.updateReportDict(summary, "error", msg)

        # Call _execute() and feed the relevant pipeline with the objects popped from requestRecords
        try:
            self.getGlobalLocks()
            totalNumRequests, cleanNumRequests, normalArchivedNumRequests, forceArchivedNumRequests = self._execute(requestRecords)
            msg = "\nNumber of processed workflows: %s."
            msg += "\nNumber of properly cleaned workflows: %s."
            msg += "\nNumber of normally archived workflows: %s."
            msg += "\nNumber of force archived workflows: %s."
            self.logger.info(msg,
                             totalNumRequests,
                             cleanNumRequests,
                             normalArchivedNumRequests,
                             forceArchivedNumRequests)
            self.updateReportDict(summary, "total_num_requests", totalNumRequests)
            self.updateReportDict(summary, "clean_num_requests", cleanNumRequests)
            self.updateReportDict(summary, "normal_archived_num_requests", normalArchivedNumRequests)
            self.updateReportDict(summary, "force_archived_num_requests", forceArchivedNumRequests)
        except Exception as ex:
            msg = "Unknown exception while running MSRuleCleaner thread Error: %s"
            self.logger.exception(msg, str(ex))
            self.updateReportDict(summary, "error", msg)

        return summary

    def _execute(self, reqRecords):
        """
        Executes the MSRuleCleaner pipelines based on the workflow status
        :param reqList: A list of RequestRecords to work on
        :return:        a tuple with:
                            number of properly cleaned requests
                            number of processed workflows
                            number of archived workflows
        """
        # NOTE: The Input Cleanup, the Block Level Cleanup and the Archival
        #       Pipelines are executed sequentially in the above order.
        #       This way we assure ourselves that we archive only workflows
        #       that have accomplished the needed cleanup

        cleanNumRequests = 0
        totalNumRequests = 0

        # Call the workflow dispatcher:
        for req in viewvalues(reqRecords):
            wflow = MSRuleCleanerWflow(req)
            self._dispatchWflow(wflow)
            msg = "\n----------------------------------------------------------"
            msg += "\nMSRuleCleanerWflow: %s"
            msg += "\n----------------------------------------------------------"
            self.logger.debug(msg, pformat(wflow))
            totalNumRequests += 1
            if self._checkClean(wflow):
                cleanNumRequests += 1

        # Report the counters:
        for pline in self.cleanuplines:
            msg = "Workflows cleaned by pipeline: %s: %d"
            self.logger.info(msg, pline.name, self.wfCounters['cleaned'][pline.name])
        normalArchivedNumRequests = self.wfCounters['archived']['normalArchived']
        forceArchivedNumRequests = self.wfCounters['archived']['forceArchived']
        self.logger.info("Workflows normally archived: %d", self.wfCounters['archived']['normalArchived'])
        self.logger.info("Workflows force archived: %d", self.wfCounters['archived']['forceArchived'])
        return totalNumRequests, cleanNumRequests, normalArchivedNumRequests, forceArchivedNumRequests

    def _dispatchWflow(self, wflow):
        """
        A function intended to dispatch a workflow (e.g based on its status)
        through one or more functional pipelines in case there is some more
        complicated logic involved in the order we execute them but not just
        a sequentially
        """
        self.logger.debug("Dispatching workflow: %s", wflow['RequestName'])
        # NOTE: The following dispatch logic is a subject to be changed at any time

        # Resolve:
        # NOTE: First resolve any preliminary flags that will be needed further
        #       in the logic of the _dispatcher() itself
        if wflow['RequestStatus'] == 'announced':
            self.getMSOutputTransferInfo(wflow)

        # Clean:
        # Do not clean any Resubmission, but still let them be archived
        if wflow['RequestType'] == 'Resubmission':
            wflow['ForceArchive'] = True
            msg = "Skipping cleanup step for workflow: %s - RequestType is %s."
            msg += " Will try to archive it directly."
            self.logger.info(msg, wflow['RequestName'], wflow['RequestType'])
        elif wflow['RequestStatus'] in ['rejected', 'aborted-completed']:
            # NOTE: We do not check the ParentageResolved flag for these
            #       workflows, but we do need to clean output data placement
            #       rules from the agents for them
            for pline in self.agentlines:
                try:
                    pline.run(wflow)
                except Exception as ex:
                    msg = "%s: General error from pipeline. Workflow: %s. Error: \n%s. "
                    msg += "\nWill retry again in the next cycle."
                    self.logger.exception(msg, pline.name, wflow['RequestName'], str(ex))
                    continue
                if wflow['CleanupStatus'][pline.name]:
                    self.wfCounters['cleaned'][pline.name] += 1
        elif wflow['RequestStatus'] == 'announced' and not wflow['ParentageResolved']:
            # NOTE: We skip workflows which are not having 'ParentageResolved'
            #       flag, but we still need some proper logging for them.
            msg = "Skipping workflow: %s - 'ParentageResolved' flag set to false."
            msg += " Will retry again in the next cycle."
            self.logger.info(msg, wflow['RequestName'])
        elif wflow['RequestStatus'] == 'announced' and not wflow['TransferDone']:
            # NOTE: We skip workflows which have not yet finalised their TransferStatus
            #       in MSOutput, but we still need some proper logging for them.
            msg = "Skipping workflow: %s - 'TransferStatus' is 'pending' or 'TransferInfo' is missing in MSOutput."
            msg += " Will retry again in the next cycle."
            self.logger.info(msg, wflow['RequestName'])
        elif wflow['RequestStatus'] == 'announced' and not wflow['TransferTape']:
            # NOTE: We skip workflows which have not yet finalised their tape transfers.
            #       (i.e. even if a single output which is supposed to be covered
            #       by a tape rule is in any of the following transient states:
            #       {REPLICATING, STUCK, SUSPENDED, WAITING_APPROVAL}.)
            #       We still need some proper logging for them.
            msg = "Skipping workflow: %s - tape transfers are not yet completed."
            msg += " Will retry again in the next cycle."
            self.logger.info(msg, wflow['RequestName'])
        elif wflow['RequestStatus'] == 'announced':
            for pline in self.cleanuplines:
                try:
                    pline.run(wflow)
                except MSRuleCleanerResolveParentError as ex:
                    msg = "%s: Parentage Resolve Error: %s. "
                    msg += "Will retry again in the next cycle."
                    self.logger.error(msg, pline.name, str(ex))
                    continue
                except Exception as ex:
                    msg = "%s: General error from pipeline. Workflow: %s. Error:  \n%s. "
                    msg += "\nWill retry again in the next cycle."
                    self.logger.exception(msg, pline.name, wflow['RequestName'], str(ex))
                    continue
                if wflow['CleanupStatus'][pline.name]:
                    self.wfCounters['cleaned'][pline.name] += 1
        else:
            # We shouldn't be here:
            msg = "Skipping workflow: %s - "
            msg += "Does not fall under any of the defined categories."
            self.logger.error(msg, wflow['RequestName'])

        # Archive:
        try:
            self.plineArchive.run(wflow)
            if wflow['ForceArchive']:
                self.wfCounters['archived']['forceArchived'] += 1
            else:
                self.wfCounters['archived']['normalArchived'] += 1
        except MSRuleCleanerArchivalSkip as ex:
            msg = "%s: Proper conditions not met: %s. "
            msg += "Skipping archival in the current cycle."
            self.logger.info(msg, wflow['PlineMarkers'][-1], str(ex))
        except MSRuleCleanerArchivalError as ex:
            msg = "%s: Archival Error: %s. "
            msg += "Will retry again in the next cycle."
            self.logger.error(msg, wflow['PlineMarkers'][-1], str(ex))
        except Exception as ex:
            msg = "%s General error from pipeline. Workflow: %s. Error: \n%s. "
            msg += "\nWill retry again in the next cycle."
            self.logger.exception(msg, wflow['PlineMarkers'][-1], wflow['RequestName'], str(ex))

    def setPlineMarker(self, wflow, pName):
        """
        A function intended to mark which is the pipeline currently working
        on the workflow. It is supposed to be called always as a first function
        in the pipeline.
        :param  wflow:   A MSRuleCleaner workflow representation
        :param  pName:   The name of the functional pipeline
        :return:         The workflow object
        """
        # NOTE: The current functional pipeline MUST always be appended at the
        #       end of the 'PlineMarkers' list

        # First get rid of the default:
        if not wflow['PlineMarkers']:
            wflow['PlineMarkers'] = []

        # Then push our current value into the markers list:
        wflow['PlineMarkers'].append(pName)

        # Populate the list of flags to be used later:
        if pName not in wflow['RulesToClean']:
            if pName in self.cleanupPipeNames:
                wflow['RulesToClean'][pName] = []
        if pName not in wflow['CleanupStatus']:
            if pName in self.cleanupPipeNames:
                wflow['CleanupStatus'][pName] = False
        return wflow

    def _checkClean(self, wflow):
        """
        An auxiliary function used to only check the temporary cleanup status.
        It basically takes the pipelines registered in 'PlineMarkers' that have
        already worked on the workflow as a mask and applies this mask over
        the set of flags in the 'CleanupStatus' field and then reduces the
        result to a single bool value
        """
        # NOTE: This is one of the few functions taking a workflow as an argument
        #       but returning a bool, since it is an auxiliary function and is not
        #       supposed to be called as a standalone function in a pipeline.
        # NOTE: `all([]) == True`, ergo all the 'rejected' && 'aborted-completed' workflows
        #       are also counted as properly cleaned and can trigger archival later

        # Build a list of bool flags based on the mask of PlineMarkers
        cleanFlagsList = [wflow['CleanupStatus'][key]
                          for key in wflow['PlineMarkers']
                          if key in wflow['CleanupStatus']]

        # If no one have worked on the workflow set the clean status to false
        if not wflow['PlineMarkers']:
            cleanStatus = False
        # If we have a mask longer than the list of flags avoid false positives
        # because of the behavior explained above - `all([]) == True`
        elif not cleanFlagsList:
            cleanStatus = False
        # Figure out the final value
        else:
            cleanStatus = all(cleanFlagsList)
        return cleanStatus

    def setClean(self, wflow):
        """
        A function to set the 'IsClean' flag based on the status from all the
        pipelines which have worked on the workflow (and have put their markers
        in the 'PlineMarkers' list)
        :param  wflow:      A MSRuleCleaner workflow representation
        :return:            The workflow object
        """
        wflow['IsClean'] = self._checkClean(wflow)
        return wflow

    def _checkLogDBClean(self, wflow):
        """
        An auxiliary function used to only check the LogDB cleanup status.
        It makes a query to LogDB in order to verify there are no any records for
        the current workflow
        :param wflow:       A MSRuleCleaner workflow representation
        :return:            True if no records were found in LogDB about wflow
        """
        cleanStatus = False
        logDBRecords = self.logDB.get(wflow['RequestName'])
        self.logger.debug("logDBRecords: %s", pformat(logDBRecords))
        if not logDBRecords:
            cleanStatus = True
        return cleanStatus

    def setLogDBClean(self, wflow):
        """
        A function to set the 'IsLogDBClean' flag based on the presence of any
        records in LogDB for the current workflow.
        :param  wflow:      A MSRuleCleaner workflow representation
        :return:            The workflow object
        """
        wflow['IsLogDBClean'] = self._checkLogDBClean(wflow)
        if not wflow['IsLogDBClean'] and wflow['IsArchivalDelayExpired']:
            wflow['IsLogDBClean'] = self._cleanLogDB(wflow)
        return wflow

    def _cleanLogDB(self, wflow):
        """
        A function to be used for cleaning all the records related to a workflow in logDB.
        :param wflow:       A MSRuleCleaner workflow representation
        :return:            True if NO errors were encountered while deleting
                            records from LogDB
        """
        cleanStatus = False
        try:
            if self.msConfig['enableRealMode']:
                self.logger.info("Deleting %s records from LogDB WMStats...", wflow['RequestName'])
                res = self.logDB.delete(wflow['RequestName'], agent=False)
                if res == 'delete-error':
                    msg = "Failed to delete logDB docs for wflow: %s" % wflow['RequestName']
                    raise MSRuleCleanerArchivalError(msg)
                cleanStatus = True
            else:
                self.logger.info("DRY-RUN: NOT Deleting %s records from LogDB WMStats...", wflow['RequestName'])
        except Exception as ex:
            msg = "General Exception while cleaning LogDB records for wflow: %s : %s"
            self.logger.exception(msg, wflow['RequestName'], str(ex))
        return cleanStatus

    def findTargetStatus(self, wflow):
        """
        Find the proper targeted archival status
        :param  wflow:      A MSRuleCleaner workflow representation
        :return:            The workflow object
        """
        # Check the available status transitions before we decide the final status
        targetStatusList = RequestStatus.REQUEST_STATE_TRANSITION.get(wflow['RequestStatus'], [])
        for status in targetStatusList:
            if self.targetStatusRegex.match(status):
                wflow['TargetStatus'] = status
        self.logger.debug("TargetStatus: %s", wflow['TargetStatus'])
        return wflow

    def _checkArchDelayExpired(self, wflow):
        """
        A function to check Archival Expiration Delay based on the information
        returned by WMStatsServer regarding the time of the last request status transition
        :param wflow:      MSRuleCleaner workflow representation
        :return:           True if the archival delay have been expired
        """
        archDelayExpired = False
        currentTime = int(time.time())
        threshold = self.msConfig['archiveDelayHours'] * 3600
        try:
            lastTransitionTime = wflow['RequestTransition'][-1]['UpdateTime']
            if lastTransitionTime and (currentTime - lastTransitionTime) > threshold:
                archDelayExpired = True
        except KeyError:
            self.logger.debug("Could not find status transition history for %s", wflow['RequestName'])
        return archDelayExpired

    def setArchivalDelayExpired(self, wflow):
        """
        A function to set the 'IsArchivalDelayExpired' flag
        """
        wflow['IsArchivalDelayExpired'] = self._checkArchDelayExpired(wflow)
        return wflow

    def archive(self, wflow):
        """
        Move the workflow to the proper archived status after checking
        the full cleanup status
        :param  wflow:      A MSRuleCleaner workflow representation
        :return:            The workflow object
        """
        # Make all the needed checks before trying to archive
        if not (wflow['IsClean'] or wflow['ForceArchive']):
            msg = "Not properly cleaned workflow: %s" % wflow['RequestName']
            raise MSRuleCleanerArchivalSkip(msg)
        if not wflow['TargetStatus']:
            msg = "Could not determine which archival status to target for workflow: %s" % wflow['RequestName']
            raise MSRuleCleanerArchivalError(msg)
        if not wflow['IsLogDBClean']:
            msg = "LogDB records have not been cleaned for workflow: %s" % wflow['RequestName']
            raise MSRuleCleanerArchivalSkip(msg)
        if not wflow['IsArchivalDelayExpired']:
            msg = "Archival delay period has not yet expired for workflow: %s." % wflow['RequestName']
            raise MSRuleCleanerArchivalSkip(msg)
        if not self.msConfig['enableRealMode']:
            msg = "Real Run Mode not enabled."
            raise MSRuleCleanerArchivalSkip(msg)

        # Proceed with the actual archival:
        try:
            self.reqmgr2.updateRequestStatus(wflow['RequestName'], wflow['TargetStatus'])
            msg = "Successful status transition to: %s for workflow: %s"
            self.logger.info(msg, wflow['TargetStatus'], wflow['RequestName'])
        except Exception as ex:
            msg = "General Exception while trying status transition to: %s " % wflow['TargetStatus']
            msg += "for workflow: %s : %s" % (wflow['RequestName'], str(ex))
            raise MSRuleCleanerArchivalError(msg)
        return wflow

    def getMSOutputTransferInfo(self, wflow):
        """
        Fetches the transfer information from the MSOutput REST interface for
        the given workflow.
        :param  wflow:   A MSRuleCleaner workflow representation
        :return:         The workflow object
        """
        headers = {'Accept': 'application/json'}
        params = {}
        url = '%s/data/info?request=%s' % (self.msConfig['msOutputUrl'],
                                           wflow['RequestName'])
        try:
            res = self.curlMgr.getdata(url, params=params, headers=headers, ckey=ckey(), cert=cert())
            data = json.loads(res)['result'][0]
            transferInfo = data['transferDoc']
        except Exception as ex:
            msg = "General exception while fetching TransferInfo from MSOutput for %s. "
            msg += "Error: %s"
            self.logger.exception(msg, wflow['RequestName'], str(ex))

        # Set Transfer status - information fetched from MSOutput only
        if transferInfo is not None and transferInfo['TransferStatus'] == 'done':
            wflow['TransferDone'] = True

        # Set Tape rules status - information fetched from Rucio (tape rule ids from MSOutput)
        if transferInfo is not None and transferInfo['OutputMap']:
            tapeRulesStatusList = []
            # For setting 'TransferTape' = True we require either no tape rules for the
            # workflow have been created or all existing tape rules to be in status 'OK',
            # so every empty TapeRuleID we consider as completed.
            for mapRecord in transferInfo['OutputMap']:
                if not mapRecord['TapeRuleID']:
                    continue
                rucioRule = self.rucio.getRule(mapRecord['TapeRuleID'])
                if not rucioRule:
                    tapeRulesStatusList.append(False)
                    msg = "Tape rule: %s not found for workflow: %s "
                    msg += "Possible server side error."
                    self.logger.error(msg, mapRecord['TapeRuleID'], wflow['RequestName'])
                    continue
                if rucioRule['state'] == 'OK':
                    tapeRulesStatusList.append(True)
                    msg = "Tape rule: %s in final state: %s for workflow: %s"
                    self.logger.info(msg, mapRecord['TapeRuleID'], rucioRule['state'], wflow['RequestName'])
                else:
                    tapeRulesStatusList.append(False)
                    msg = "Tape rule: %s in non final state: %s for workflow: %s"
                    self.logger.info(msg, mapRecord['TapeRuleID'], rucioRule['state'], wflow['RequestName'])
            if all(tapeRulesStatusList):
                wflow['TransferTape'] = True

        return wflow

    def setParentDatasets(self, wflow):
        """
        Used to resolve parent datasets for a workflow.
        :param  wflow:   A MSRuleCleaner workflow representation
        :return:         The workflow object
        """
        if wflow['InputDataset'] and wflow['IncludeParents']:
            childDataset = wflow['InputDataset']
            parentDataset = findParent([childDataset], self.msConfig['dbsUrl'])
            # NOTE: If findParent() returned None then the DBS service failed to
            #       resolve the request (it is considered an ERROR outside WMCore)
            if parentDataset.get(childDataset, None) is None:
                msg = "Failed to resolve parent dataset for: %s in workflow: %s" % (childDataset, wflow['RequestName'])
                raise MSRuleCleanerResolveParentError(msg)
            elif parentDataset:
                wflow['ParentDataset'] = [parentDataset[childDataset]]
                msg = "Found parent %s for input dataset %s in workflow: %s "
                self.logger.info(msg, parentDataset, wflow['InputDataset'], wflow['RequestName'])
            else:
                msg = "Could not find parent for input dataset: %s in workflows: %s"
                self.logger.error(msg, wflow['InputDataset'], wflow['RequestName'])
        return wflow

    def getRucioRules(self, wflow, gran, rucioAcct):
        """
        Queries Rucio and builds the relevant list of blocklevel rules for
        the given workflow
        :param  wflow:   A MSRuleCleaner workflow representation
        :param  gran:    Data granularity to search for Rucio rules. Possible values:
                         'block' or 'container'
        :return:         The workflow object
        """
        currPline = wflow['PlineMarkers'][-1]

        # Create the container list to the rucio account map and set the checkGlobalLocks flag.
        mapRuleType = {self.msConfig['rucioWmaAccount']: ["OutputDatasets"],
                       self.msConfig['rucioMStrAccount']: ["InputDataset", "MCPileup",
                                                           "DataPileup", "ParentDataset"]}
        if rucioAcct == self.msConfig['rucioMStrAccount']:
            checkGlobalLocks = True
        else:
            checkGlobalLocks = False

        # Find all the data placement rules created by the components:
        for dataType in mapRuleType[rucioAcct]:
            dataList = wflow[dataType] if isinstance(wflow[dataType], list) else [wflow[dataType]]
            for dataCont in dataList:
                if dataCont is None:
                    continue
                self.logger.debug("getRucioRules: dataCont: %s", pformat(dataCont))
                if checkGlobalLocks and dataCont in self.globalLocks:
                    msg = "Found dataset: %s in GlobalLocks. NOT considering it for filling the "
                    msg += "RulesToClean list for both container and block level Rules for workflow: %s!"
                    self.logger.info(msg, dataCont, wflow['RequestName'])
                    continue
                if gran == 'container':
                    for rule in self.rucio.listDataRules(dataCont, account=rucioAcct):
                        wflow['RulesToClean'][currPline].append(rule['id'])
                        msg = "Found %s container-level rule to be deleted for container %s"
                        self.logger.info(msg, rule['id'], dataCont)
                elif gran == 'block':
                    try:
                        blocks = self.rucio.getBlocksInContainer(dataCont)
                        for block in blocks:
                            for rule in self.rucio.listDataRules(block, account=rucioAcct):
                                wflow['RulesToClean'][currPline].append(rule['id'])
                                msg = "Found %s block-level rule to be deleted for container %s"
                                self.logger.info(msg, rule['id'], dataCont)
                    except WMRucioDIDNotFoundException:
                        msg = "Container: %s not found in Rucio for workflow: %s."
                        self.logger.info(msg, dataCont, wflow['RequestName'])
        return wflow

    def cleanRucioRules(self, wflow):
        """
        Cleans all the Rules present in the field 'RulesToClean' in the MSRuleCleaner
        workflow representation. And fills the relevant Cleanup Status.
        :param wflow:   A MSRuleCleaner workflow representation
        :return:        The workflow object
        """
        # NOTE: The function should be called independently and sequentially from
        #       The Input and the respective BlockLevel pipelines.

        # NOTE: The current functional pipeline is always the last one in the PlineMarkers list
        currPline = wflow['PlineMarkers'][-1]
        delResults = []
        if self.msConfig['enableRealMode']:
            for rule in wflow['RulesToClean'][currPline]:
                self.logger.info("%s: Deleting ruleId: %s ", currPline, rule)
                delResult = self.rucio.deleteRule(rule)
                delResults.append(delResult)
                if not delResult:
                    self.logger.warning("%s: Failed to delete ruleId: %s ", currPline, rule)
        else:
            for rule in wflow['RulesToClean'][currPline]:
                delResults.append(True)
                self.logger.info("%s: DRY-RUN: Is about to delete ruleId: %s ", currPline, rule)

        # Set the cleanup flag:
        wflow['CleanupStatus'][currPline] = all(delResults)
        return wflow

    def getRequestRecords(self, reqStatus):
        """
        Queries ReqMgr2 for requests in a given status.
        :param reqStatus: The status for the requests to be fetched from ReqMgr2
        :return requests: A dictionary with all the workflows in the given status
        """
        self.logger.info("Fetching requests in status: %s", reqStatus)
        result = self.reqmgr2.getRequestByStatus([reqStatus], detail=True)
        if not result:
            requests = {}
        else:
            requests = result[0]
        self.logger.info('  retrieved %s requests in status: %s', len(requests), reqStatus)
        return requests
Example #50
0
    def __init__(self, msConfig, logger=None):
        """
        Runs the basic setup and initialization for the MSRuleCleaner module
        :param msConfig: micro service configuration
        """
        super(MSRuleCleaner, self).__init__(msConfig, logger=logger)

        self.msConfig.setdefault("verbose", True)
        self.msConfig.setdefault("interval", 60)
        self.msConfig.setdefault("services", ['ruleCleaner'])
        self.msConfig.setdefault("rucioWmaAccount", "wma_test")
        self.msConfig.setdefault("rucioMStrAccount", "wmcore_transferor")
        self.msConfig.setdefault('enableRealMode', False)
        self.mode = "RealMode" if self.msConfig[
            'enableRealMode'] else "DryRunMode"
        self.emailAlert = EmailAlert(self.msConfig)
        self.curlMgr = RequestHandler()

        # Building all the Pipelines:
        pName = 'plineMSTrCont'
        self.plineMSTrCont = Pipeline(name=pName,
                                      funcLine=[
                                          Functor(self.setPlineMarker, pName),
                                          Functor(self.cleanRucioRules)
                                      ])
        pName = 'plineMSTrBlock'
        self.plineMSTrBlock = Pipeline(name=pName,
                                       funcLine=[
                                           Functor(self.setPlineMarker, pName),
                                           Functor(self.cleanRucioRules)
                                       ])
        pName = 'plineAgentCont'
        self.plineAgentCont = Pipeline(
            name=pName,
            funcLine=[
                Functor(self.setPlineMarker, pName),
                Functor(self.getRucioRules, 'container',
                        self.msConfig['rucioWmaAccount']),
                Functor(self.cleanRucioRules)
            ])
        pName = 'plineAgentBlock'
        self.plineAgentBlock = Pipeline(
            name=pName,
            funcLine=[
                Functor(self.setPlineMarker, pName),
                Functor(self.getRucioRules, 'block',
                        self.msConfig['rucioWmaAccount']),
                Functor(self.cleanRucioRules)
            ])
        pName = 'plineArchive'
        self.plineArchive = Pipeline(name=pName,
                                     funcLine=[
                                         Functor(self.setPlineMarker, pName),
                                         Functor(self.setClean),
                                         Functor(self.archive)
                                     ])

        # Building the different set of plines we will need later:
        # NOTE: The following are all the functional pipelines which are supposed to include
        #       a cleanup function and report cleanup status in the MSRuleCleanerWflow object
        self.cleanuplines = [
            self.plineMSTrCont, self.plineMSTrBlock, self.plineAgentCont,
            self.plineAgentBlock
        ]
        # Building an auxiliary list of cleanup pipeline names only:
        self.cleanupPipeNames = [pline.name for pline in self.cleanuplines]

        # Building lists of pipelines related only to Agents or MStransferror
        self.agentlines = [self.plineAgentCont, self.plineAgentBlock]
        self.mstrlines = [self.plineMSTrCont, self.plineMSTrBlock]

        # Initialization of the 'cleaned' and 'archived' counters:
        self.wfCounters = {'cleaned': {}, 'archived': 0}
Example #51
0
    def command(self, jobs, jobs_lfn, jobs_pfn, jobs_report):
        """
        For each job the worker has to complete:
        Delete files that have failed previously
        Create a temporary copyjob file
        Submit the copyjob to the appropriate FTS server
        Parse the output of the FTS transfer and return complete and failed files for recording
        """
        # Output: {"userProxyPath":"/path/to/proxy","LFNs":["lfn1","lfn2","lfn3"],"PFNs":["pfn1","pfn2","pfn3"],"FTSJobid":'id-of-fts-job', "username": '******'}
        #Loop through all the jobs for the links we have
        failure_reasons = []
        for link, copyjob in jobs.items():
            submission_error = False
            status_error = False
            fts_job = {}
            # Validate copyjob file before doing anything
            self.logger.debug("Valid %s" % self.validate_copyjob(copyjob))
            if not self.validate_copyjob(copyjob): continue

            rest_copyjob = {
                        "params":{
                                "bring_online": None,
                                "verify_checksum": False,
                                "copy_pin_lifetime": -1,
                                "max_time_in_queue": self.config.max_h_in_queue,
                                "job_metadata":{"issuer": "ASO"},
                                "spacetoken": None,
                                "source_spacetoken": None,
                                "fail_nearline": False,
                                "overwrite": True,
                                "gridftp": None
                        },
                        "files":[]
                }

            pairs = []
            for SrcDest in copyjob:
                tempDict = {"sources": [], "metadata": None, "destinations": []}

                tempDict["sources"].append(SrcDest.split(" ")[0])
                tempDict["destinations"].append(SrcDest.split(" ")[1])
                rest_copyjob["files"].append(tempDict)


            self.logger.debug("Subbmitting this REST copyjob %s" % rest_copyjob)
            url = self.fts_server_for_transfer + '/jobs'
            self.logger.debug("Running FTS submission command")
            self.logger.debug("FTS server: %s" % self.fts_server_for_transfer)
            self.logger.debug("link: %s -> %s" % link)
            heade = {"Content-Type ":"application/json"}
            buf = StringIO.StringIO()
            try:
                connection = RequestHandler(config={'timeout': 300, 'connecttimeout' : 300})
            except Exception as ex:
                msg = str(ex)
                msg += str(traceback.format_exc())
                self.logger.debug(msg)
            try:
                response, datares = connection.request(url, rest_copyjob, heade, verb='POST', doseq=True, ckey=self.user_proxy, \
                                                       cert=self.user_proxy, capath='/etc/grid-security/certificates', \
                                                       cainfo=self.user_proxy, verbose=True)
                self.logger.debug("Submission done")
                self.logger.debug('Submission header status: %s' % response.status)
                self.logger.debug('Submission header reason: %s' % response.reason)
                self.logger.debug('Submission result %s' %  datares)
            except Exception as ex:
                msg = "Error submitting to FTS: %s " % url
                msg += str(ex)
                msg += str(traceback.format_exc())
                self.logger.debug(msg)
                failure_reasons.append(msg)
                submission_error = True
            buf.close()
            if not submission_error:
                res = {}
                try:
                    res = json.loads(datares)
                except Exception as ex:
                    msg = "Couldn't load submission acknowledgment from FTS"
                    msg += str(ex)
                    msg += str(traceback.format_exc())
                    self.logger.debug(msg)
                    submission_error = True
                    failure_reasons.append(msg)
                if 'job_id' in res:
                    fileId_list = []
                    files_res = []
                    files_ = {}
                    job_id = res['job_id']
                    file_url = self.fts_server_for_transfer + '/jobs/' + job_id +'/files'
                    self.logger.debug("Submitting to %s" % file_url)
                    file_buf = StringIO.StringIO()
                    try:
                        response, files_ = connection.request(file_url, {}, heade, doseq=True, ckey=self.user_proxy, \
                                                              cert=self.user_proxy, capath='/etc/grid-security/certificates', \
                                                              cainfo=self.user_proxy, verbose=True)
                        files_res = json.loads(files_)
                    except Exception as ex:
                        msg = "Error contacting FTS to retrieve file: %s " % file_url
                        msg += str(ex)
                        msg += str(traceback.format_exc())
                        self.logger.debug(msg)
                        submission_error = True
                        failure_reasons.append(msg)
                    self.logger.debug("List files in job %s" % files_)
                    file_buf.close()
                    for file_in_job in files_res:
                        if 'file_id' in file_in_job:
                            fileId_list.append(file_in_job['file_id'])
                        else:
                            msg = "Could not load submitted file %s from FTS" % file_url
                            self.logger.debug(msg)
                            submission_error = True
                            failure_reasons.append(msg)
                    self.logger.debug("File id list %s" % fileId_list)
            if submission_error:
                self.logger.debug("Submission failed")
                self.logger.info("Mark failed %s files" % len(jobs_lfn[link]))
                self.logger.debug("Mark failed %s files" % jobs_lfn[link])
                failed_files = self.mark_failed(jobs_lfn[link], force_fail=False, submission_error=True, failure_reasons=failure_reasons)
                self.logger.info("Marked failed %s" % len(failed_files))
                continue
            fts_job['userProxyPath'] = self.user_proxy
            fts_job['LFNs'] = jobs_lfn[link]
            fts_job['PFNs'] = jobs_pfn[link]
            fts_job['FTSJobid'] = job_id
            fts_job['files_id'] = fileId_list
            fts_job['username'] = self.user
            self.logger.debug("Creating json file %s in %s" % (fts_job, self.dropbox_dir))
            ftsjob_file = open('%s/Monitor.%s.json' % (self.dropbox_dir, fts_job['FTSJobid']), 'w')
            jsondata = json.dumps(fts_job)
            ftsjob_file.write(jsondata)
            ftsjob_file.close()
            self.logger.debug("%s ready." % fts_job)
            # Prepare Dashboard report
            for lfn in fts_job['LFNs']:
                lfn_report = {}
                lfn_report['FTSJobid'] = fts_job['FTSJobid']
                index = fts_job['LFNs'].index(lfn)
                lfn_report['PFN'] = fts_job['PFNs'][index]
                lfn_report['FTSFileid'] = fts_job['files_id'][index]
                lfn_report['Workflow'] = jobs_report[link][index][2]
                lfn_report['JobVersion'] = jobs_report[link][index][1]
                job_id = '%d_https://glidein.cern.ch/%d/%s_%s' % (int(jobs_report[link][index][0]), int(jobs_report[link][index][0]), lfn_report['Workflow'].replace("_", ":"), lfn_report['JobVersion'])
                lfn_report['JobId'] = job_id
                lfn_report['URL'] = self.fts_server_for_transfer
                self.logger.debug("Creating json file %s in %s for FTS3 Dashboard" % (lfn_report, self.dropbox_dir))
                dash_job_file = open('/tmp/Dashboard.%s.json' % getHashLfn(lfn_report['PFN']), 'w')
                jsondata = json.dumps(lfn_report)
                dash_job_file.write(jsondata)
                dash_job_file.close()
                self.logger.debug("%s ready for FTS Dashboard report." % lfn_report)
        return