def load(self, filename): """ _load_ UncPickle data from file """ #TODO: currently support both loading from file path or url #if there are more things to filter may be separate the load function # urllib2 needs a scheme - assume local file if none given if not urlparse(filename)[0]: filename = 'file:' + filename handle = urlopen(Request(filename, headers = {"Accept" : "*/*"})) self.data = cPickle.load(handle) handle.close() elif filename.startswith('file:'): handle = urlopen(Request(filename, headers = {"Accept" : "*/*"})) self.data = cPickle.load(handle) handle.close() else: # use own request class so we get authentication if needed from WMCore.Services.Requests import Requests request = Requests(filename) data = request.makeRequest('', incoming_headers = {"Accept" : "*/*"}) self.data = cPickle.loads(data[0]) #TODO: use different encoding scheme for different extension #extension = filename.split(".")[-1].lower() return
def check_resident(self, site, stage_data): """ Checks that the requested data is resident on the site """ goodToGo = True # Format site name locSite = site.replace('_Buffer', '_MSS') if not locSite.endswith('_MSS'): locSite += '_MSS' qParams = {'node': locSite} # Get block info from PhEDEx phedex = Requests(url='https://cmsweb.cern.ch', idict={'accept_type':'application/json'}) for data in stage_data: if data.find('#') < 0: qParams['dataset'] = data else: qParams['block'] = data self.logger.debug('Checking data residency for %s at %s' % (data, locSite)) try: pdata = phedex.get('/phedex/datasvc/json/prod/blockreplicas', qParams)[0] except httplib.HTTPException, he: self.handleHTTPExcept(he, 'HTTPException for block: %s node: %s' % (data, locSite)) # Parse block info and check > 0 block exist try: if len(JsonWrapper.loads(pdata)['phedex']['block']) == 0: goodToGo = False self.logger.error('Block %s not resident at site %s' % (data, locSite)) except: self.logger.debug('error loading json') goodToGo = False
def load(self, filename): """ _load_ UncPickle data from file """ #TODO: currently support both loading from file path or url #if there are more things to filter may be separate the load function # urllib2 needs a scheme - assume local file if none given if not urlparse(filename)[0]: filename = 'file:' + filename handle = urlopen(Request(filename, headers={"Accept": "*/*"})) self.data = cPickle.load(handle) handle.close() elif filename.startswith('file:'): handle = urlopen(Request(filename, headers={"Accept": "*/*"})) self.data = cPickle.load(handle) handle.close() else: # use own request class so we get authentication if needed from WMCore.Services.Requests import Requests request = Requests(filename) data = request.makeRequest('', incoming_headers={"Accept": "*/*"}) self.data = cPickle.loads(data[0]) #TODO: use different encoding scheme for different extension #extension = filename.split(".")[-1].lower() return
class PyCurlRESTServer(RESTBaseUnitTest): """ Loads the rest server and test the upload """ def initialize(self): self.config = DefaultConfig('WMCore_t.Services_t.PyCurlRESTModel') self.config.Webtools.environment = 'development' self.config.Webtools.error_log_level = logging.ERROR self.config.Webtools.access_log_level = logging.ERROR self.config.Webtools.port = 8888 self.config.Webtools.host = '127.0.0.1' self.config.UnitTests.object = 'WMCore_t.Services_t.PyCurlRESTModel' self.requestHandler = Requests('http://127.0.0.1:8888/unittests/rest/') # This test is flipping back and forth in Jenkins. Perhaps due to port 8888 not being available. # Disabling for now @attr("integration") def testFileUpload(self): """ The method upload a file (data/TestUpload.txt) and check if the server API has saved it """ uploadedFilename = 'UploadedFile.txt' fileName = os.path.join(getTestBase(), 'WMCore_t/Services_t/TestUpload.txt') #Check the uploaded file is not there if os.path.isfile(uploadedFilename): os.remove(uploadedFilename) self.assertFalse(os.path.isfile(uploadedFilename)) #do the upload res = self.requestHandler.uploadFile( fileName, 'http://127.0.0.1:8888/unittests/rest/file/') #the file is there now (?) self.assertTrue(os.path.isfile(uploadedFilename)) self.assertEquals(open(uploadedFilename).read(), open(fileName).read()) #delete the uploaded file os.remove(uploadedFilename) self.assertTrue('Success' in res) # This test is flipping back and forth in Jenkins. Perhaps due to port 8888 not being available. # Disabling for now @attr("integration") def testFailingFileUpload(self): """ The method upload a file (data/TestUpload.txt) and check if the server API has saved it """ uploadedFilename = 'UploadedFile.txt' fileName = os.path.join(getTestBase(), 'WMCore_t/Services_t/TestUpload.txt') #Check the uploaded file is not there if os.path.isfile(uploadedFilename): os.remove(uploadedFilename) self.assertFalse(os.path.isfile(uploadedFilename)) #do the upload using the wrong address try: res = self.requestHandler.uploadFile( fileName, 'http://127.0.0.1:8888/unittests/rest/iAmNotThere/') self.fail() except HTTPException as he: self.assertEqual(he.status, 404) self.assertFalse(os.path.isfile(uploadedFilename))
def initialize(self): self.config = DefaultConfig('WMCore_t.Services_t.PyCurlRESTModel') self.config.Webtools.environment = 'development' self.config.Webtools.error_log_level = logging.ERROR self.config.Webtools.access_log_level = logging.ERROR self.config.Webtools.port = 8888 self.config.Webtools.host = '127.0.0.1' self.config.UnitTests.object = 'WMCore_t.Services_t.PyCurlRESTModel' self.requestHandler = Requests('http://127.0.0.1:8888/unittests/rest/')
class PyCurlRESTServer(RESTBaseUnitTest): """ Loads the rest server and test the upload """ def initialize(self): self.config = DefaultConfig('WMCore_t.Services_t.PyCurlRESTModel') self.config.Webtools.environment = 'development' self.config.Webtools.error_log_level = logging.ERROR self.config.Webtools.access_log_level = logging.ERROR self.config.Webtools.port = 8888 self.config.Webtools.host = '127.0.0.1' self.config.UnitTests.object = 'WMCore_t.Services_t.PyCurlRESTModel' self.requestHandler = Requests('http://127.0.0.1:8888/unittests/rest/') # This test is flipping back and forth in Jenkins. Perhaps due to port 8888 not being available. # Disabling for now @attr("integration") def testFileUpload(self): """ The method upload a file (data/TestUpload.txt) and check if the server API has saved it """ uploadedFilename = 'UploadedFile.txt' fileName = os.path.join(getTestBase(), 'WMCore_t/Services_t/TestUpload.txt') #Check the uploaded file is not there if os.path.isfile(uploadedFilename): os.remove(uploadedFilename) self.assertFalse( os.path.isfile(uploadedFilename)) #do the upload res = self.requestHandler.uploadFile(fileName, 'http://127.0.0.1:8888/unittests/rest/file/') #the file is there now (?) self.assertTrue( os.path.isfile(uploadedFilename)) self.assertEquals( open(uploadedFilename).read() , open(fileName).read() ) #delete the uploaded file os.remove(uploadedFilename) self.assertTrue('Success' in res) # This test is flipping back and forth in Jenkins. Perhaps due to port 8888 not being available. # Disabling for now @attr("integration") def testFailingFileUpload(self): """ The method upload a file (data/TestUpload.txt) and check if the server API has saved it """ uploadedFilename = 'UploadedFile.txt' fileName = os.path.join(getTestBase(), 'WMCore_t/Services_t/TestUpload.txt') #Check the uploaded file is not there if os.path.isfile(uploadedFilename): os.remove(uploadedFilename) self.assertFalse( os.path.isfile(uploadedFilename)) #do the upload using the wrong address try: res = self.requestHandler.uploadFile(fileName, 'http://127.0.0.1:8888/unittests/rest/iAmNotThere/') self.fail() except HTTPException as he: self.assertEqual(he.status, 404) self.assertFalse( os.path.isfile(uploadedFilename))
def process_files(self, stage_data = [], request_id=''): """ Contact PhEDEx data service to get a list of files for a given request. TODO: use Service.PhEDEx """ db = self.localcouch.connectDatabase('%s_stagequeue' % self.site) # TODO: make the phedex URL a configurable! phedex = Requests(url='https://cmsweb.cern.ch', idict={'accept_type':'application/json'}) totalFiles = 0 totalBytes = 0 # Need to clean up the input a bit for d in stage_data: qParams = {'node': self.node} # Need to pass in blocks if d.find('#') < 0: qParams['dataset'] = d else: qParams['block'] = d try: pdata = phedex.get('/phedex/datasvc/json/prod/filereplicas', qParams)[0] except httplib.HTTPException, he: self.handleHTTPExcept(he, 'HTTPException for block: %s node: %s' % (d, self.node)) # Parse block info try: p = JsonWrapper.loads(pdata) if len(p['phedex']['block']) == 0: self.logger.error('Block %s not resident at site %s' % (d, self.node)) else: self.logger.debug('Creating stage-in requests for %s' % self.node) for blk in p['phedex']['block']: self.logger.debug('Creating stage-in requests for block %s' % blk['name']) for fle in blk['file']: self.logger.debug('Creating stage-in requests for file %s' % fle['name']) checksum = fle['checksum'] f = {'_id': fle['name'], 'bytes': int(fle['bytes']), 'checksum': {checksum.split(':')[0]: checksum.split(':')[1]}, 'state': 'new', 'retry_count': [], 'request_id': request_id } try: db.queue(f, timestamp = True, viewlist=['stagequeue/file_state']) totalFiles += 1 totalBytes += f['bytes'] except httplib.HTTPException, he: self.handleHTTPExcept(he, 'Could not commit data') except: self.logger.debug('error loading json')
def testExecute(self): #recycle DataDiscoveryTest code to create the input of this test ddObj, task, requestname, datasetfiles, locations = DataDiscoveryTest.prepareObjects( ) res = ddObj.formatOutput(task=task, requestname=requestname, datasetfiles=datasetfiles, locations=locations) #Test the case where the lumimask is empty. Thats the most interesting case cert, key = Requests().getKeyCert() server = HTTPRequests(os.environ['REST_URL'], cert, key, version="0.debug") lmb = LumiMaskBuilder(None, server, "/crabserver/dev/workflowdb") task = {} task['tm_split_args'] = {} #this is a wf name I had in the REST db. Used to check by hand if the db was updated. #we should create a RESTMock for unit tests task['tm_taskname'] = "130719_090932_mmascher_crab_tmp" task['tm_split_args']['lumis'] = {} task['tm_split_args']['runs'] = {} lmb.execute(res.result, task=task) self.assertEqual(lmb.runs, ['1', '2', '3', '4']) self.assertEqual(lmb.lumis[1:], ['1,5,8,9,20,22', '11,13', '1,2,5,7,100,100' ]) #first run too long to check in a unit test
def wmbsServiceSetup(self, argstring, kargs={}, returnType='text'): if returnType == 'json': request = JSONRequests(self.server_url) else: request = Requests(self.server_url) results = request.get("/wmbs/%s/" % argstring, kargs) return results
def testBaadGetDefault(self): req = Requests('localhost:8308') self.runReq(self.badurls, req, 400)
def testGoodGetDefault(self): req = Requests('localhost:8308') self.runReq(self.goodurls, req, 200)
def prepareObjects(): #Creting the input parameters requestname = '130911_093053_mmascher_crab_tmp2' task = { 'tm_taskname': requestname } #just using some locations for this dataset locations = { '/GenericTTbar/HC-CMSSW_5_3_1_START53_V5-v1/GEN-SIM-RECO#289639b0-146f-411b-a220-fcdff0a13cd9': [ u'ganymede.hep.kbfi.ee', u'srm.ihepa.ufl.edu', u'storm-se-01.ba.infn.it' ] } #and just using some files datasetfiles = {} datasetfiles[ '/store/mc/HC/GenericTTbar/GEN-SIM-RECO/CMSSW_5_3_1_START53_V5-v1/0011/626F3DC8-1EAE-E111-A64C-0025902CB6B0.root'] = { 'NumberOfEvents': 1700L, 'BlockName': '/GenericTTbar/HC-CMSSW_5_3_1_START53_V5-v1/GEN-SIM-RECO#289639b0-146f-411b-a220-fcdff0a13cd9', 'Lumis': { 1L: [ 670994L, 670986L, 670958L, 671001L, 670995L, 671018L, 670988L, 671009L, 671038L, 671028L, 670942L, 670960L, 670948L, 670987L, 670991L, 671006L, 671012L, 671014L, 671040L, 670998L, 670983L, 670999L, 671027L, 671052L, 670984L, 671003L, 670996L, 671031L, 671007L, 671011L, 671022L, 670985L, 670962L, 671021L ], 2L: [1L, 2L, 3L, 8L, 9L], 4L: [5L, 6L, 7L, 1L, 2L, 100L] }, 'Parents': [ '/store/mc/HC/GenericTTbar/GEN-SIM-RAW/CMSSW_5_3_1_START53_V5-v1/0011/C880E78D-1EAE-E111-897B-0025902CB6AE.root', '/store/mc/HC/GenericTTbar/GEN-SIM-RAW/CMSSW_5_3_1_START53_V5-v1/0011/8C7BB8D5-1EAE-E111-B269-0025901AD638.root', '/store/mc/HC/GenericTTbar/GEN-SIM-RAW/CMSSW_5_3_1_START53_V5-v1/0011/8215258E-1EAE-E111-8A4E-0025904B11D4.root' ], 'Checksums': { 'Checksum': '922282544', 'Adler32': '39f2938b', 'Md5': 'NOTSET' }, 'Size': 975355735L } datasetfiles[ '/store/mc/HC/GenericTTbar/GEN-SIM-RECO/CMSSW_5_3_1_START53_V5-v1/0011/7A5634E8-03AE-E111-9363-5404A63886EC.root'] = { 'NumberOfEvents': 1700L, 'BlockName': '/GenericTTbar/HC-CMSSW_5_3_1_START53_V5-v1/GEN-SIM-RECO#289639b0-146f-411b-a220-fcdff0a13cd9', 'Lumis': { 1L: [ 670165L, 670174L, 670199L, 670081L, 670192L, 670170L, 670029L, 670093L, 670173L, 670164L, 670146L, 670079L, 670200L, 670065L, 670062L, 670216L, 670202L, 670150L, 670176L, 670207L, 670087L, 670179L, 670191L, 670185L, 670160L, 670182L, 670177L, 670203L, 670154L, 670187L, 670138L, 670162L, 670066L, 670085L ], 2L: [4L, 5L, 20L, 21L, 22L], 3L: [11L, 12L, 13L] }, 'Parents': [ '/store/mc/HC/GenericTTbar/GEN-SIM-RAW/CMSSW_5_3_1_START53_V5-v1/0011/BAC6F21E-04AE-E111-A591-003048D3C90E.root', '/store/mc/HC/GenericTTbar/GEN-SIM-RAW/CMSSW_5_3_1_START53_V5-v1/0011/781547E6-03AE-E111-B1C0-BCAEC5329717.root', '/store/mc/HC/GenericTTbar/GEN-SIM-RAW/CMSSW_5_3_1_START53_V5-v1/0011/38694250-03AE-E111-B618-003048D37580.root' ], 'Checksums': { 'Checksum': '2628876232', 'Adler32': '21d69fc7', 'Md5': 'NOTSET' }, 'Size': 974920274L } #Instantiate DataDiscovery config = Configuration() config.section_("MyProxy") config.MyProxy.serverhostcert, config.MyProxy.serverhostkey = Requests( ).getKeyCert() obj = DataDiscovery(config, None, None) return obj, task, requestname, datasetfiles, locations