Beispiel #1
0
    def load(self, filename):
        """
        _load_

        UncPickle data from file

        """

        #TODO: currently support both loading from file path or url
        #if there are more things to filter may be separate the load function

        # urllib2 needs a scheme - assume local file if none given
        if not urlparse(filename)[0]:
            filename = 'file:' + filename
            handle = urlopen(Request(filename, headers = {"Accept" : "*/*"}))
            self.data = cPickle.load(handle)
            handle.close()
        elif filename.startswith('file:'):
            handle = urlopen(Request(filename, headers = {"Accept" : "*/*"}))
            self.data = cPickle.load(handle)
            handle.close()
        else:
            # use own request class so we get authentication if needed
            from WMCore.Services.Requests import Requests
            request = Requests(filename)
            data = request.makeRequest('', incoming_headers = {"Accept" : "*/*"})
            self.data = cPickle.loads(data[0])

        #TODO: use different encoding scheme for different extension
        #extension = filename.split(".")[-1].lower()

        return
    def check_resident(self, site, stage_data):
        """
        Checks that the requested data is resident on the site
        """
        goodToGo = True

        # Format site name
        locSite = site.replace('_Buffer', '_MSS')
        if not locSite.endswith('_MSS'):
            locSite += '_MSS'
        qParams = {'node': locSite}

        # Get block info from PhEDEx
        phedex = Requests(url='https://cmsweb.cern.ch', idict={'accept_type':'application/json'})
        for data in stage_data:
            if data.find('#') < 0:
                qParams['dataset'] = data
            else:
                qParams['block'] = data

            self.logger.debug('Checking data residency for %s at %s' % (data, locSite))
            try:
                pdata = phedex.get('/phedex/datasvc/json/prod/blockreplicas', qParams)[0]
            except httplib.HTTPException, he:
                self.handleHTTPExcept(he, 'HTTPException for block: %s node: %s' % (data, locSite))

            # Parse block info and check > 0 block exist
            try:
                if len(JsonWrapper.loads(pdata)['phedex']['block']) == 0:
                   goodToGo = False
                   self.logger.error('Block %s not resident at site %s' % (data, locSite))
            except:
                self.logger.debug('error loading json')
                goodToGo = False
Beispiel #3
0
    def load(self, filename):
        """
        _load_

        UncPickle data from file

        """

        #TODO: currently support both loading from file path or url
        #if there are more things to filter may be separate the load function

        # urllib2 needs a scheme - assume local file if none given
        if not urlparse(filename)[0]:
            filename = 'file:' + filename
            handle = urlopen(Request(filename, headers={"Accept": "*/*"}))
            self.data = cPickle.load(handle)
            handle.close()
        elif filename.startswith('file:'):
            handle = urlopen(Request(filename, headers={"Accept": "*/*"}))
            self.data = cPickle.load(handle)
            handle.close()
        else:
            # use own request class so we get authentication if needed
            from WMCore.Services.Requests import Requests
            request = Requests(filename)
            data = request.makeRequest('', incoming_headers={"Accept": "*/*"})
            self.data = cPickle.loads(data[0])

        #TODO: use different encoding scheme for different extension
        #extension = filename.split(".")[-1].lower()

        return
Beispiel #4
0
class PyCurlRESTServer(RESTBaseUnitTest):
    """
    Loads the rest server and test the upload
    """
    def initialize(self):
        self.config = DefaultConfig('WMCore_t.Services_t.PyCurlRESTModel')
        self.config.Webtools.environment = 'development'
        self.config.Webtools.error_log_level = logging.ERROR
        self.config.Webtools.access_log_level = logging.ERROR
        self.config.Webtools.port = 8888
        self.config.Webtools.host = '127.0.0.1'
        self.config.UnitTests.object = 'WMCore_t.Services_t.PyCurlRESTModel'
        self.requestHandler = Requests('http://127.0.0.1:8888/unittests/rest/')

    # This test is flipping back and forth in Jenkins. Perhaps due to port 8888 not being available.
    # Disabling for now
    @attr("integration")
    def testFileUpload(self):
        """
        The method upload a file (data/TestUpload.txt) and check if the server API has saved it
        """
        uploadedFilename = 'UploadedFile.txt'
        fileName = os.path.join(getTestBase(),
                                'WMCore_t/Services_t/TestUpload.txt')
        #Check the uploaded file is not there
        if os.path.isfile(uploadedFilename):
            os.remove(uploadedFilename)
            self.assertFalse(os.path.isfile(uploadedFilename))
        #do the upload
        res = self.requestHandler.uploadFile(
            fileName, 'http://127.0.0.1:8888/unittests/rest/file/')
        #the file is there now (?)
        self.assertTrue(os.path.isfile(uploadedFilename))
        self.assertEquals(open(uploadedFilename).read(), open(fileName).read())
        #delete the uploaded file
        os.remove(uploadedFilename)
        self.assertTrue('Success' in res)

    # This test is flipping back and forth in Jenkins. Perhaps due to port 8888 not being available.
    # Disabling for now
    @attr("integration")
    def testFailingFileUpload(self):
        """
        The method upload a file (data/TestUpload.txt) and check if the server API has saved it
        """
        uploadedFilename = 'UploadedFile.txt'
        fileName = os.path.join(getTestBase(),
                                'WMCore_t/Services_t/TestUpload.txt')
        #Check the uploaded file is not there
        if os.path.isfile(uploadedFilename):
            os.remove(uploadedFilename)
            self.assertFalse(os.path.isfile(uploadedFilename))
        #do the upload using the wrong address
        try:
            res = self.requestHandler.uploadFile(
                fileName, 'http://127.0.0.1:8888/unittests/rest/iAmNotThere/')
            self.fail()
        except HTTPException as he:
            self.assertEqual(he.status, 404)
        self.assertFalse(os.path.isfile(uploadedFilename))
Beispiel #5
0
 def initialize(self):
     self.config = DefaultConfig('WMCore_t.Services_t.PyCurlRESTModel')
     self.config.Webtools.environment = 'development'
     self.config.Webtools.error_log_level = logging.ERROR
     self.config.Webtools.access_log_level = logging.ERROR
     self.config.Webtools.port = 8888
     self.config.Webtools.host = '127.0.0.1'
     self.config.UnitTests.object = 'WMCore_t.Services_t.PyCurlRESTModel'
     self.requestHandler = Requests('http://127.0.0.1:8888/unittests/rest/')
Beispiel #6
0
class PyCurlRESTServer(RESTBaseUnitTest):
    """
    Loads the rest server and test the upload
    """

    def initialize(self):
        self.config = DefaultConfig('WMCore_t.Services_t.PyCurlRESTModel')
        self.config.Webtools.environment = 'development'
        self.config.Webtools.error_log_level = logging.ERROR
        self.config.Webtools.access_log_level = logging.ERROR
        self.config.Webtools.port = 8888
        self.config.Webtools.host = '127.0.0.1'
        self.config.UnitTests.object = 'WMCore_t.Services_t.PyCurlRESTModel'
        self.requestHandler = Requests('http://127.0.0.1:8888/unittests/rest/')

    # This test is flipping back and forth in Jenkins. Perhaps due to port 8888 not being available.
    # Disabling for now
    @attr("integration")
    def testFileUpload(self):
        """
        The method upload a file (data/TestUpload.txt) and check if the server API has saved it
        """
        uploadedFilename = 'UploadedFile.txt'
        fileName = os.path.join(getTestBase(), 'WMCore_t/Services_t/TestUpload.txt')
        #Check the uploaded file is not there
        if os.path.isfile(uploadedFilename):
            os.remove(uploadedFilename)
            self.assertFalse( os.path.isfile(uploadedFilename))
        #do the upload
        res = self.requestHandler.uploadFile(fileName, 'http://127.0.0.1:8888/unittests/rest/file/')
        #the file is there now (?)
        self.assertTrue( os.path.isfile(uploadedFilename))
        self.assertEquals( open(uploadedFilename).read() , open(fileName).read() )
        #delete the uploaded file
        os.remove(uploadedFilename)
        self.assertTrue('Success' in res)

    # This test is flipping back and forth in Jenkins. Perhaps due to port 8888 not being available.
    # Disabling for now
    @attr("integration")
    def testFailingFileUpload(self):
        """
        The method upload a file (data/TestUpload.txt) and check if the server API has saved it
        """
        uploadedFilename = 'UploadedFile.txt'
        fileName = os.path.join(getTestBase(), 'WMCore_t/Services_t/TestUpload.txt')
        #Check the uploaded file is not there
        if os.path.isfile(uploadedFilename):
            os.remove(uploadedFilename)
            self.assertFalse( os.path.isfile(uploadedFilename))
        #do the upload using the wrong address
        try:
            res = self.requestHandler.uploadFile(fileName, 'http://127.0.0.1:8888/unittests/rest/iAmNotThere/')
            self.fail()
        except HTTPException as he:
            self.assertEqual(he.status, 404)
        self.assertFalse( os.path.isfile(uploadedFilename))
Beispiel #7
0
    def process_files(self, stage_data = [], request_id=''):
        """
        Contact PhEDEx data service to get a list of files for a given request.
        TODO: use Service.PhEDEx
        """
        db = self.localcouch.connectDatabase('%s_stagequeue' % self.site)

        # TODO: make the phedex URL a configurable!
        phedex = Requests(url='https://cmsweb.cern.ch', idict={'accept_type':'application/json'})

        totalFiles = 0
        totalBytes = 0
        # Need to clean up the input a bit
        for d in stage_data:
            qParams = {'node': self.node}
            # Need to pass in blocks
            if d.find('#') < 0:
                qParams['dataset'] = d
            else:
                qParams['block'] = d

            try:
               pdata = phedex.get('/phedex/datasvc/json/prod/filereplicas', qParams)[0]
            except httplib.HTTPException, he:
               self.handleHTTPExcept(he, 'HTTPException for block: %s node: %s' % (d, self.node))

            # Parse block info
            try:
                p = JsonWrapper.loads(pdata)
                if len(p['phedex']['block']) == 0:
                   self.logger.error('Block %s not resident at site %s' % (d, self.node))
                else:
                   self.logger.debug('Creating stage-in requests for %s' % self.node)
                   for blk in p['phedex']['block']:
                      self.logger.debug('Creating stage-in requests for block %s' % blk['name'])
                      for fle in blk['file']:
                         self.logger.debug('Creating stage-in requests for file %s' % fle['name'])
                         checksum = fle['checksum']
                         f = {'_id': fle['name'],
                              'bytes': int(fle['bytes']),
                              'checksum': {checksum.split(':')[0]: checksum.split(':')[1]},
                              'state': 'new',
                              'retry_count': [],
                              'request_id': request_id
                         }
                         try:
                            db.queue(f, timestamp = True, viewlist=['stagequeue/file_state'])
                            totalFiles += 1
                            totalBytes += f['bytes']
                         except httplib.HTTPException, he:
                            self.handleHTTPExcept(he, 'Could not commit data')
            except:
                self.logger.debug('error loading json')
    def testExecute(self):
        #recycle DataDiscoveryTest code to create the input of this test
        ddObj, task, requestname, datasetfiles, locations = DataDiscoveryTest.prepareObjects(
        )
        res = ddObj.formatOutput(task=task,
                                 requestname=requestname,
                                 datasetfiles=datasetfiles,
                                 locations=locations)

        #Test the case where the lumimask is empty. Thats the most interesting case
        cert, key = Requests().getKeyCert()
        server = HTTPRequests(os.environ['REST_URL'],
                              cert,
                              key,
                              version="0.debug")
        lmb = LumiMaskBuilder(None, server, "/crabserver/dev/workflowdb")

        task = {}
        task['tm_split_args'] = {}
        #this is a wf name I had in the REST db. Used to check by hand if the db was updated.
        #we should create a RESTMock for unit tests
        task['tm_taskname'] = "130719_090932_mmascher_crab_tmp"
        task['tm_split_args']['lumis'] = {}
        task['tm_split_args']['runs'] = {}
        lmb.execute(res.result, task=task)

        self.assertEqual(lmb.runs, ['1', '2', '3', '4'])
        self.assertEqual(lmb.lumis[1:],
                         ['1,5,8,9,20,22', '11,13', '1,2,5,7,100,100'
                          ])  #first run too long to check in a unit test
 def initialize(self):
     self.config = DefaultConfig('WMCore_t.Services_t.PyCurlRESTModel')
     self.config.Webtools.environment = 'development'
     self.config.Webtools.error_log_level = logging.ERROR
     self.config.Webtools.access_log_level = logging.ERROR
     self.config.Webtools.port = 8888
     self.config.Webtools.host = '127.0.0.1'
     self.config.UnitTests.object = 'WMCore_t.Services_t.PyCurlRESTModel'
     self.requestHandler = Requests('http://127.0.0.1:8888/unittests/rest/')
Beispiel #10
0
    def wmbsServiceSetup(self, argstring, kargs={}, returnType='text'):

        if returnType == 'json':
            request = JSONRequests(self.server_url)
        else:
            request = Requests(self.server_url)
        results = request.get("/wmbs/%s/" % argstring, kargs)

        return results
Beispiel #11
0
 def testBaadGetDefault(self):
     req = Requests('localhost:8308')
     self.runReq(self.badurls, req, 400)
Beispiel #12
0
 def testGoodGetDefault(self):
     req = Requests('localhost:8308')
     self.runReq(self.goodurls, req, 200)
    def prepareObjects():
        #Creting the input parameters
        requestname = '130911_093053_mmascher_crab_tmp2'
        task = {
            'tm_taskname': requestname
        }  #just using some locations for this dataset
        locations = {
            '/GenericTTbar/HC-CMSSW_5_3_1_START53_V5-v1/GEN-SIM-RECO#289639b0-146f-411b-a220-fcdff0a13cd9':
            [
                u'ganymede.hep.kbfi.ee', u'srm.ihepa.ufl.edu',
                u'storm-se-01.ba.infn.it'
            ]
        }
        #and just using some files
        datasetfiles = {}
        datasetfiles[
            '/store/mc/HC/GenericTTbar/GEN-SIM-RECO/CMSSW_5_3_1_START53_V5-v1/0011/626F3DC8-1EAE-E111-A64C-0025902CB6B0.root'] = {
                'NumberOfEvents':
                1700L,
                'BlockName':
                '/GenericTTbar/HC-CMSSW_5_3_1_START53_V5-v1/GEN-SIM-RECO#289639b0-146f-411b-a220-fcdff0a13cd9',
                'Lumis': {
                    1L: [
                        670994L, 670986L, 670958L, 671001L, 670995L, 671018L,
                        670988L, 671009L, 671038L, 671028L, 670942L, 670960L,
                        670948L, 670987L, 670991L, 671006L, 671012L, 671014L,
                        671040L, 670998L, 670983L, 670999L, 671027L, 671052L,
                        670984L, 671003L, 670996L, 671031L, 671007L, 671011L,
                        671022L, 670985L, 670962L, 671021L
                    ],
                    2L: [1L, 2L, 3L, 8L, 9L],
                    4L: [5L, 6L, 7L, 1L, 2L, 100L]
                },
                'Parents': [
                    '/store/mc/HC/GenericTTbar/GEN-SIM-RAW/CMSSW_5_3_1_START53_V5-v1/0011/C880E78D-1EAE-E111-897B-0025902CB6AE.root',
                    '/store/mc/HC/GenericTTbar/GEN-SIM-RAW/CMSSW_5_3_1_START53_V5-v1/0011/8C7BB8D5-1EAE-E111-B269-0025901AD638.root',
                    '/store/mc/HC/GenericTTbar/GEN-SIM-RAW/CMSSW_5_3_1_START53_V5-v1/0011/8215258E-1EAE-E111-8A4E-0025904B11D4.root'
                ],
                'Checksums': {
                    'Checksum': '922282544',
                    'Adler32': '39f2938b',
                    'Md5': 'NOTSET'
                },
                'Size':
                975355735L
            }
        datasetfiles[
            '/store/mc/HC/GenericTTbar/GEN-SIM-RECO/CMSSW_5_3_1_START53_V5-v1/0011/7A5634E8-03AE-E111-9363-5404A63886EC.root'] = {
                'NumberOfEvents':
                1700L,
                'BlockName':
                '/GenericTTbar/HC-CMSSW_5_3_1_START53_V5-v1/GEN-SIM-RECO#289639b0-146f-411b-a220-fcdff0a13cd9',
                'Lumis': {
                    1L: [
                        670165L, 670174L, 670199L, 670081L, 670192L, 670170L,
                        670029L, 670093L, 670173L, 670164L, 670146L, 670079L,
                        670200L, 670065L, 670062L, 670216L, 670202L, 670150L,
                        670176L, 670207L, 670087L, 670179L, 670191L, 670185L,
                        670160L, 670182L, 670177L, 670203L, 670154L, 670187L,
                        670138L, 670162L, 670066L, 670085L
                    ],
                    2L: [4L, 5L, 20L, 21L, 22L],
                    3L: [11L, 12L, 13L]
                },
                'Parents': [
                    '/store/mc/HC/GenericTTbar/GEN-SIM-RAW/CMSSW_5_3_1_START53_V5-v1/0011/BAC6F21E-04AE-E111-A591-003048D3C90E.root',
                    '/store/mc/HC/GenericTTbar/GEN-SIM-RAW/CMSSW_5_3_1_START53_V5-v1/0011/781547E6-03AE-E111-B1C0-BCAEC5329717.root',
                    '/store/mc/HC/GenericTTbar/GEN-SIM-RAW/CMSSW_5_3_1_START53_V5-v1/0011/38694250-03AE-E111-B618-003048D37580.root'
                ],
                'Checksums': {
                    'Checksum': '2628876232',
                    'Adler32': '21d69fc7',
                    'Md5': 'NOTSET'
                },
                'Size':
                974920274L
            }
        #Instantiate DataDiscovery
        config = Configuration()
        config.section_("MyProxy")
        config.MyProxy.serverhostcert, config.MyProxy.serverhostkey = Requests(
        ).getKeyCert()
        obj = DataDiscovery(config, None, None)

        return obj, task, requestname, datasetfiles, locations