def load(self, filename):
"""
_load_
UncPickle data from file
"""
#TODO: currently support both loading from file path or url
#if there are more things to filter may be separate the load function
# urllib2 needs a scheme - assume local file if none given
if not urlparse(filename)[0]:
filename = 'file:' + filename
handle = urlopen(Request(filename, headers = {"Accept" : "*/*"}))
self.data = cPickle.load(handle)
handle.close()
elif filename.startswith('file:'):
handle = urlopen(Request(filename, headers = {"Accept" : "*/*"}))
self.data = cPickle.load(handle)
handle.close()
else:
# use own request class so we get authentication if needed
from WMCore.Services.Requests import Requests
request = Requests(filename)
data = request.makeRequest('', incoming_headers = {"Accept" : "*/*"})
self.data = cPickle.loads(data[0])
#TODO: use different encoding scheme for different extension
#extension = filename.split(".")[-1].lower()
return
def check_resident(self, site, stage_data):
"""
Checks that the requested data is resident on the site
"""
goodToGo = True
# Format site name
locSite = site.replace('_Buffer', '_MSS')
if not locSite.endswith('_MSS'):
locSite += '_MSS'
qParams = {'node': locSite}
# Get block info from PhEDEx
phedex = Requests(url='https://cmsweb.cern.ch', idict={'accept_type':'application/json'})
for data in stage_data:
if data.find('#') < 0:
qParams['dataset'] = data
else:
qParams['block'] = data
self.logger.debug('Checking data residency for %s at %s' % (data, locSite))
try:
pdata = phedex.get('/phedex/datasvc/json/prod/blockreplicas', qParams)[0]
except httplib.HTTPException, he:
self.handleHTTPExcept(he, 'HTTPException for block: %s node: %s' % (data, locSite))
# Parse block info and check > 0 block exist
try:
if len(JsonWrapper.loads(pdata)['phedex']['block']) == 0:
goodToGo = False
self.logger.error('Block %s not resident at site %s' % (data, locSite))
except:
self.logger.debug('error loading json')
goodToGo = False
def load(self, filename):
"""
_load_
UncPickle data from file
"""
#TODO: currently support both loading from file path or url
#if there are more things to filter may be separate the load function
# urllib2 needs a scheme - assume local file if none given
if not urlparse(filename)[0]:
filename = 'file:' + filename
handle = urlopen(Request(filename, headers={"Accept": "*/*"}))
self.data = cPickle.load(handle)
handle.close()
elif filename.startswith('file:'):
handle = urlopen(Request(filename, headers={"Accept": "*/*"}))
self.data = cPickle.load(handle)
handle.close()
else:
# use own request class so we get authentication if needed
from WMCore.Services.Requests import Requests
request = Requests(filename)
data = request.makeRequest('', incoming_headers={"Accept": "*/*"})
self.data = cPickle.loads(data[0])
#TODO: use different encoding scheme for different extension
#extension = filename.split(".")[-1].lower()
return
class PyCurlRESTServer(RESTBaseUnitTest):
"""
Loads the rest server and test the upload
"""
def initialize(self):
self.config = DefaultConfig('WMCore_t.Services_t.PyCurlRESTModel')
self.config.Webtools.environment = 'development'
self.config.Webtools.error_log_level = logging.ERROR
self.config.Webtools.access_log_level = logging.ERROR
self.config.Webtools.port = 8888
self.config.Webtools.host = '127.0.0.1'
self.config.UnitTests.object = 'WMCore_t.Services_t.PyCurlRESTModel'
self.requestHandler = Requests('http://127.0.0.1:8888/unittests/rest/')
# This test is flipping back and forth in Jenkins. Perhaps due to port 8888 not being available.
# Disabling for now
@attr("integration")
def testFileUpload(self):
"""
The method upload a file (data/TestUpload.txt) and check if the server API has saved it
"""
uploadedFilename = 'UploadedFile.txt'
fileName = os.path.join(getTestBase(),
'WMCore_t/Services_t/TestUpload.txt')
#Check the uploaded file is not there
if os.path.isfile(uploadedFilename):
os.remove(uploadedFilename)
self.assertFalse(os.path.isfile(uploadedFilename))
#do the upload
res = self.requestHandler.uploadFile(
fileName, 'http://127.0.0.1:8888/unittests/rest/file/')
#the file is there now (?)
self.assertTrue(os.path.isfile(uploadedFilename))
self.assertEquals(open(uploadedFilename).read(), open(fileName).read())
#delete the uploaded file
os.remove(uploadedFilename)
self.assertTrue('Success' in res)
# This test is flipping back and forth in Jenkins. Perhaps due to port 8888 not being available.
# Disabling for now
@attr("integration")
def testFailingFileUpload(self):
"""
The method upload a file (data/TestUpload.txt) and check if the server API has saved it
"""
uploadedFilename = 'UploadedFile.txt'
fileName = os.path.join(getTestBase(),
'WMCore_t/Services_t/TestUpload.txt')
#Check the uploaded file is not there
if os.path.isfile(uploadedFilename):
os.remove(uploadedFilename)
self.assertFalse(os.path.isfile(uploadedFilename))
#do the upload using the wrong address
try:
res = self.requestHandler.uploadFile(
fileName, 'http://127.0.0.1:8888/unittests/rest/iAmNotThere/')
self.fail()
except HTTPException as he:
self.assertEqual(he.status, 404)
self.assertFalse(os.path.isfile(uploadedFilename))
def initialize(self):
self.config = DefaultConfig('WMCore_t.Services_t.PyCurlRESTModel')
self.config.Webtools.environment = 'development'
self.config.Webtools.error_log_level = logging.ERROR
self.config.Webtools.access_log_level = logging.ERROR
self.config.Webtools.port = 8888
self.config.Webtools.host = '127.0.0.1'
self.config.UnitTests.object = 'WMCore_t.Services_t.PyCurlRESTModel'
self.requestHandler = Requests('http://127.0.0.1:8888/unittests/rest/')
class PyCurlRESTServer(RESTBaseUnitTest):
"""
Loads the rest server and test the upload
"""
def initialize(self):
self.config = DefaultConfig('WMCore_t.Services_t.PyCurlRESTModel')
self.config.Webtools.environment = 'development'
self.config.Webtools.error_log_level = logging.ERROR
self.config.Webtools.access_log_level = logging.ERROR
self.config.Webtools.port = 8888
self.config.Webtools.host = '127.0.0.1'
self.config.UnitTests.object = 'WMCore_t.Services_t.PyCurlRESTModel'
self.requestHandler = Requests('http://127.0.0.1:8888/unittests/rest/')
# This test is flipping back and forth in Jenkins. Perhaps due to port 8888 not being available.
# Disabling for now
@attr("integration")
def testFileUpload(self):
"""
The method upload a file (data/TestUpload.txt) and check if the server API has saved it
"""
uploadedFilename = 'UploadedFile.txt'
fileName = os.path.join(getTestBase(), 'WMCore_t/Services_t/TestUpload.txt')
#Check the uploaded file is not there
if os.path.isfile(uploadedFilename):
os.remove(uploadedFilename)
self.assertFalse( os.path.isfile(uploadedFilename))
#do the upload
res = self.requestHandler.uploadFile(fileName, 'http://127.0.0.1:8888/unittests/rest/file/')
#the file is there now (?)
self.assertTrue( os.path.isfile(uploadedFilename))
self.assertEquals( open(uploadedFilename).read() , open(fileName).read() )
#delete the uploaded file
os.remove(uploadedFilename)
self.assertTrue('Success' in res)
# This test is flipping back and forth in Jenkins. Perhaps due to port 8888 not being available.
# Disabling for now
@attr("integration")
def testFailingFileUpload(self):
"""
The method upload a file (data/TestUpload.txt) and check if the server API has saved it
"""
uploadedFilename = 'UploadedFile.txt'
fileName = os.path.join(getTestBase(), 'WMCore_t/Services_t/TestUpload.txt')
#Check the uploaded file is not there
if os.path.isfile(uploadedFilename):
os.remove(uploadedFilename)
self.assertFalse( os.path.isfile(uploadedFilename))
#do the upload using the wrong address
try:
res = self.requestHandler.uploadFile(fileName, 'http://127.0.0.1:8888/unittests/rest/iAmNotThere/')
self.fail()
except HTTPException as he:
self.assertEqual(he.status, 404)
self.assertFalse( os.path.isfile(uploadedFilename))
def process_files(self, stage_data = [], request_id=''):
"""
Contact PhEDEx data service to get a list of files for a given request.
TODO: use Service.PhEDEx
"""
db = self.localcouch.connectDatabase('%s_stagequeue' % self.site)
# TODO: make the phedex URL a configurable!
phedex = Requests(url='https://cmsweb.cern.ch', idict={'accept_type':'application/json'})
totalFiles = 0
totalBytes = 0
# Need to clean up the input a bit
for d in stage_data:
qParams = {'node': self.node}
# Need to pass in blocks
if d.find('#') < 0:
qParams['dataset'] = d
else:
qParams['block'] = d
try:
pdata = phedex.get('/phedex/datasvc/json/prod/filereplicas', qParams)[0]
except httplib.HTTPException, he:
self.handleHTTPExcept(he, 'HTTPException for block: %s node: %s' % (d, self.node))
# Parse block info
try:
p = JsonWrapper.loads(pdata)
if len(p['phedex']['block']) == 0:
self.logger.error('Block %s not resident at site %s' % (d, self.node))
else:
self.logger.debug('Creating stage-in requests for %s' % self.node)
for blk in p['phedex']['block']:
self.logger.debug('Creating stage-in requests for block %s' % blk['name'])
for fle in blk['file']:
self.logger.debug('Creating stage-in requests for file %s' % fle['name'])
checksum = fle['checksum']
f = {'_id': fle['name'],
'bytes': int(fle['bytes']),
'checksum': {checksum.split(':')[0]: checksum.split(':')[1]},
'state': 'new',
'retry_count': [],
'request_id': request_id
}
try:
db.queue(f, timestamp = True, viewlist=['stagequeue/file_state'])
totalFiles += 1
totalBytes += f['bytes']
except httplib.HTTPException, he:
self.handleHTTPExcept(he, 'Could not commit data')
except:
self.logger.debug('error loading json')
def testExecute(self):
#recycle DataDiscoveryTest code to create the input of this test
ddObj, task, requestname, datasetfiles, locations = DataDiscoveryTest.prepareObjects(
)
res = ddObj.formatOutput(task=task,
requestname=requestname,
datasetfiles=datasetfiles,
locations=locations)
#Test the case where the lumimask is empty. Thats the most interesting case
cert, key = Requests().getKeyCert()
server = HTTPRequests(os.environ['REST_URL'],
cert,
key,
version="0.debug")
lmb = LumiMaskBuilder(None, server, "/crabserver/dev/workflowdb")
task = {}
task['tm_split_args'] = {}
#this is a wf name I had in the REST db. Used to check by hand if the db was updated.
#we should create a RESTMock for unit tests
task['tm_taskname'] = "130719_090932_mmascher_crab_tmp"
task['tm_split_args']['lumis'] = {}
task['tm_split_args']['runs'] = {}
lmb.execute(res.result, task=task)
self.assertEqual(lmb.runs, ['1', '2', '3', '4'])
self.assertEqual(lmb.lumis[1:],
['1,5,8,9,20,22', '11,13', '1,2,5,7,100,100'
]) #first run too long to check in a unit test
def initialize(self):
self.config = DefaultConfig('WMCore_t.Services_t.PyCurlRESTModel')
self.config.Webtools.environment = 'development'
self.config.Webtools.error_log_level = logging.ERROR
self.config.Webtools.access_log_level = logging.ERROR
self.config.Webtools.port = 8888
self.config.Webtools.host = '127.0.0.1'
self.config.UnitTests.object = 'WMCore_t.Services_t.PyCurlRESTModel'
self.requestHandler = Requests('http://127.0.0.1:8888/unittests/rest/')
def wmbsServiceSetup(self, argstring, kargs={}, returnType='text'):
if returnType == 'json':
request = JSONRequests(self.server_url)
else:
request = Requests(self.server_url)
results = request.get("/wmbs/%s/" % argstring, kargs)
return results
def testBaadGetDefault(self):
req = Requests('localhost:8308')
self.runReq(self.badurls, req, 400)
def testGoodGetDefault(self):
req = Requests('localhost:8308')
self.runReq(self.goodurls, req, 200)
def prepareObjects():
#Creting the input parameters
requestname = '130911_093053_mmascher_crab_tmp2'
task = {
'tm_taskname': requestname
} #just using some locations for this dataset
locations = {
'/GenericTTbar/HC-CMSSW_5_3_1_START53_V5-v1/GEN-SIM-RECO#289639b0-146f-411b-a220-fcdff0a13cd9':
[
u'ganymede.hep.kbfi.ee', u'srm.ihepa.ufl.edu',
u'storm-se-01.ba.infn.it'
]
}
#and just using some files
datasetfiles = {}
datasetfiles[
'/store/mc/HC/GenericTTbar/GEN-SIM-RECO/CMSSW_5_3_1_START53_V5-v1/0011/626F3DC8-1EAE-E111-A64C-0025902CB6B0.root'] = {
'NumberOfEvents':
1700L,
'BlockName':
'/GenericTTbar/HC-CMSSW_5_3_1_START53_V5-v1/GEN-SIM-RECO#289639b0-146f-411b-a220-fcdff0a13cd9',
'Lumis': {
1L: [
670994L, 670986L, 670958L, 671001L, 670995L, 671018L,
670988L, 671009L, 671038L, 671028L, 670942L, 670960L,
670948L, 670987L, 670991L, 671006L, 671012L, 671014L,
671040L, 670998L, 670983L, 670999L, 671027L, 671052L,
670984L, 671003L, 670996L, 671031L, 671007L, 671011L,
671022L, 670985L, 670962L, 671021L
],
2L: [1L, 2L, 3L, 8L, 9L],
4L: [5L, 6L, 7L, 1L, 2L, 100L]
},
'Parents': [
'/store/mc/HC/GenericTTbar/GEN-SIM-RAW/CMSSW_5_3_1_START53_V5-v1/0011/C880E78D-1EAE-E111-897B-0025902CB6AE.root',
'/store/mc/HC/GenericTTbar/GEN-SIM-RAW/CMSSW_5_3_1_START53_V5-v1/0011/8C7BB8D5-1EAE-E111-B269-0025901AD638.root',
'/store/mc/HC/GenericTTbar/GEN-SIM-RAW/CMSSW_5_3_1_START53_V5-v1/0011/8215258E-1EAE-E111-8A4E-0025904B11D4.root'
],
'Checksums': {
'Checksum': '922282544',
'Adler32': '39f2938b',
'Md5': 'NOTSET'
},
'Size':
975355735L
}
datasetfiles[
'/store/mc/HC/GenericTTbar/GEN-SIM-RECO/CMSSW_5_3_1_START53_V5-v1/0011/7A5634E8-03AE-E111-9363-5404A63886EC.root'] = {
'NumberOfEvents':
1700L,
'BlockName':
'/GenericTTbar/HC-CMSSW_5_3_1_START53_V5-v1/GEN-SIM-RECO#289639b0-146f-411b-a220-fcdff0a13cd9',
'Lumis': {
1L: [
670165L, 670174L, 670199L, 670081L, 670192L, 670170L,
670029L, 670093L, 670173L, 670164L, 670146L, 670079L,
670200L, 670065L, 670062L, 670216L, 670202L, 670150L,
670176L, 670207L, 670087L, 670179L, 670191L, 670185L,
670160L, 670182L, 670177L, 670203L, 670154L, 670187L,
670138L, 670162L, 670066L, 670085L
],
2L: [4L, 5L, 20L, 21L, 22L],
3L: [11L, 12L, 13L]
},
'Parents': [
'/store/mc/HC/GenericTTbar/GEN-SIM-RAW/CMSSW_5_3_1_START53_V5-v1/0011/BAC6F21E-04AE-E111-A591-003048D3C90E.root',
'/store/mc/HC/GenericTTbar/GEN-SIM-RAW/CMSSW_5_3_1_START53_V5-v1/0011/781547E6-03AE-E111-B1C0-BCAEC5329717.root',
'/store/mc/HC/GenericTTbar/GEN-SIM-RAW/CMSSW_5_3_1_START53_V5-v1/0011/38694250-03AE-E111-B618-003048D37580.root'
],
'Checksums': {
'Checksum': '2628876232',
'Adler32': '21d69fc7',
'Md5': 'NOTSET'
},
'Size':
974920274L
}
#Instantiate DataDiscovery
config = Configuration()
config.section_("MyProxy")
config.MyProxy.serverhostcert, config.MyProxy.serverhostkey = Requests(
).getKeyCert()
obj = DataDiscovery(config, None, None)
return obj, task, requestname, datasetfiles, locations
