def load(self, filename): """ _load_ UncPickle data from file """ #TODO: currently support both loading from file path or url #if there are more things to filter may be separate the load function # urllib2 needs a scheme - assume local file if none given if not urlparse(filename)[0]: filename = 'file:' + filename handle = urlopen(Request(filename, headers={"Accept": "*/*"})) self.data = cPickle.load(handle) handle.close() elif filename.startswith('file:'): handle = urlopen(Request(filename, headers={"Accept": "*/*"})) self.data = cPickle.load(handle) handle.close() else: # use own request class so we get authentication if needed from WMCore.Services.Requests import Requests request = Requests(filename) data = request.makeRequest('', incoming_headers={"Accept": "*/*"}) self.data = cPickle.loads(data[0]) #TODO: use different encoding scheme for different extension #extension = filename.split(".")[-1].lower() return
def testExecute(self): #recycle DataDiscoveryTest code to create the input of this test ddObj, task, requestname, datasetfiles, locations = DataDiscoveryTest.prepareObjects( ) res = ddObj.formatOutput(task=task, requestname=requestname, datasetfiles=datasetfiles, locations=locations) #Test the case where the lumimask is empty. Thats the most interesting case cert, key = Requests().getKeyCert() server = HTTPRequests(os.environ['REST_URL'], cert, key, version="0.debug") lmb = LumiMaskBuilder(None, server, "/crabserver/dev/workflowdb") task = {} task['tm_split_args'] = {} #this is a wf name I had in the REST db. Used to check by hand if the db was updated. #we should create a RESTMock for unit tests task['tm_taskname'] = "130719_090932_mmascher_crab_tmp" task['tm_split_args']['lumis'] = {} task['tm_split_args']['runs'] = {} lmb.execute(res.result, task=task) self.assertEqual(lmb.runs, ['1', '2', '3', '4']) self.assertEqual(lmb.lumis[1:], ['1,5,8,9,20,22', '11,13', '1,2,5,7,100,100' ]) #first run too long to check in a unit test
def wmbsServiceSetup(self, argstring, kargs={}, returnType='text'): if returnType == 'json': request = JSONRequests(self.server_url) else: request = Requests(self.server_url) results = request.get("/wmbs/%s/" % argstring, kargs) return results
def initialize(self): self.config = DefaultConfig('WMCore_t.Services_t.PyCurlRESTModel') self.config.Webtools.environment = 'development' self.config.Webtools.error_log_level = logging.ERROR self.config.Webtools.access_log_level = logging.ERROR self.config.Webtools.port = 8888 self.config.Webtools.host = '127.0.0.1' self.config.UnitTests.object = 'WMCore_t.Services_t.PyCurlRESTModel' self.requestHandler = Requests('http://127.0.0.1:8888/unittests/rest/')
def testBaadGetDefault(self): req = Requests('localhost:8308') self.runReq(self.badurls, req, 400)
def testGoodGetDefault(self): req = Requests('localhost:8308') self.runReq(self.goodurls, req, 200)
def prepareObjects(): #Creting the input parameters requestname = '130911_093053_mmascher_crab_tmp2' task = { 'tm_taskname': requestname } #just using some locations for this dataset locations = { '/GenericTTbar/HC-CMSSW_5_3_1_START53_V5-v1/GEN-SIM-RECO#289639b0-146f-411b-a220-fcdff0a13cd9': [ u'ganymede.hep.kbfi.ee', u'srm.ihepa.ufl.edu', u'storm-se-01.ba.infn.it' ] } #and just using some files datasetfiles = {} datasetfiles[ '/store/mc/HC/GenericTTbar/GEN-SIM-RECO/CMSSW_5_3_1_START53_V5-v1/0011/626F3DC8-1EAE-E111-A64C-0025902CB6B0.root'] = { 'NumberOfEvents': 1700L, 'BlockName': '/GenericTTbar/HC-CMSSW_5_3_1_START53_V5-v1/GEN-SIM-RECO#289639b0-146f-411b-a220-fcdff0a13cd9', 'Lumis': { 1L: [ 670994L, 670986L, 670958L, 671001L, 670995L, 671018L, 670988L, 671009L, 671038L, 671028L, 670942L, 670960L, 670948L, 670987L, 670991L, 671006L, 671012L, 671014L, 671040L, 670998L, 670983L, 670999L, 671027L, 671052L, 670984L, 671003L, 670996L, 671031L, 671007L, 671011L, 671022L, 670985L, 670962L, 671021L ], 2L: [1L, 2L, 3L, 8L, 9L], 4L: [5L, 6L, 7L, 1L, 2L, 100L] }, 'Parents': [ '/store/mc/HC/GenericTTbar/GEN-SIM-RAW/CMSSW_5_3_1_START53_V5-v1/0011/C880E78D-1EAE-E111-897B-0025902CB6AE.root', '/store/mc/HC/GenericTTbar/GEN-SIM-RAW/CMSSW_5_3_1_START53_V5-v1/0011/8C7BB8D5-1EAE-E111-B269-0025901AD638.root', '/store/mc/HC/GenericTTbar/GEN-SIM-RAW/CMSSW_5_3_1_START53_V5-v1/0011/8215258E-1EAE-E111-8A4E-0025904B11D4.root' ], 'Checksums': { 'Checksum': '922282544', 'Adler32': '39f2938b', 'Md5': 'NOTSET' }, 'Size': 975355735L } datasetfiles[ '/store/mc/HC/GenericTTbar/GEN-SIM-RECO/CMSSW_5_3_1_START53_V5-v1/0011/7A5634E8-03AE-E111-9363-5404A63886EC.root'] = { 'NumberOfEvents': 1700L, 'BlockName': '/GenericTTbar/HC-CMSSW_5_3_1_START53_V5-v1/GEN-SIM-RECO#289639b0-146f-411b-a220-fcdff0a13cd9', 'Lumis': { 1L: [ 670165L, 670174L, 670199L, 670081L, 670192L, 670170L, 670029L, 670093L, 670173L, 670164L, 670146L, 670079L, 670200L, 670065L, 670062L, 670216L, 670202L, 670150L, 670176L, 670207L, 670087L, 670179L, 670191L, 670185L, 670160L, 670182L, 670177L, 670203L, 670154L, 670187L, 670138L, 670162L, 670066L, 670085L ], 2L: [4L, 5L, 20L, 21L, 22L], 3L: [11L, 12L, 13L] }, 'Parents': [ '/store/mc/HC/GenericTTbar/GEN-SIM-RAW/CMSSW_5_3_1_START53_V5-v1/0011/BAC6F21E-04AE-E111-A591-003048D3C90E.root', '/store/mc/HC/GenericTTbar/GEN-SIM-RAW/CMSSW_5_3_1_START53_V5-v1/0011/781547E6-03AE-E111-B1C0-BCAEC5329717.root', '/store/mc/HC/GenericTTbar/GEN-SIM-RAW/CMSSW_5_3_1_START53_V5-v1/0011/38694250-03AE-E111-B618-003048D37580.root' ], 'Checksums': { 'Checksum': '2628876232', 'Adler32': '21d69fc7', 'Md5': 'NOTSET' }, 'Size': 974920274L } #Instantiate DataDiscovery config = Configuration() config.section_("MyProxy") config.MyProxy.serverhostcert, config.MyProxy.serverhostkey = Requests( ).getKeyCert() obj = DataDiscovery(config, None, None) return obj, task, requestname, datasetfiles, locations