def getRequestInfo():
requestName = sys.argv[1]
print "Getting information from cmsweb about %s... %s" % (requestName, time.strftime('%H:%M:%S'))
requestor = JSONRequests(url = 'https://cmsweb.cern.ch/reqmgr/reqMgr')
response = requestor.get('/request?requestName=%s' % requestName)
if response[1] != 200:
raise RuntimeError("Request information was not available!")
requestDict = response[0]
#print requestDict.get('RunWhitelist')
#print requestDict.get('RunBlacklist')
if not requestDict.get('RunWhitelist'):
requestDict['RunWhitelist'] = []
if not requestDict.get('RunBlacklist'):
requestDict['RunBlacklist'] = []
if not requestDict.get('BlockWhitelist'):
requestDict['BlockWhitelist'] = []
if not requestDict.get('BlockBlacklist'):
requestDict['BlockBlacklist'] = []
compactRequest = {'InputDataset' : requestDict['InputDataset'],
'RunWhitelist' : requestDict['RunWhitelist'],
'RunBlacklist' : requestDict['RunBlacklist'],
'BlockWhitelist' : requestDict['BlockWhitelist'],
'BlockBlacklist' : requestDict['BlockBlacklist'],
'OutputDatasets' : []}
response = requestor.get('/outputDatasetsByRequestName?requestName=%s' % requestName)
if response[1] != 200:
raise RuntimeError("Output dataset information was not available!")
compactRequest['OutputDatasets'] = response[0]
print "Done querying cmsweb... %s" % time.strftime('%H:%M:%S')
return compactRequest
def setUp(self):
RESTBaseUnitTest.setUp(self)
self.testInit.setupCouch("%s" % self.couchDBName, "GroupUser",
"ConfigCache", "ReqMgr")
self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats")
# logging stuff from TestInit is broken, setting myself
l = logging.getLogger()
l.setLevel(logging.DEBUG)
self.params = {}
self.params['endpoint'] = self.config.getServerUrl()
self.reqService = RequestManagerDS(self.params)
self.jsonSender = JSONRequests(self.config.getServerUrl())
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName=userName,
groupName=groupName,
teamName=teamName)
schema['ConfigCacheID'] = self.createConfig()
schema['CouchDBName'] = self.couchDBName
schema['CouchWorkloadDBName'] = self.couchDBName
try:
r = self.jsonSender.put('request', schema)
try:
self.requestName = r[0]['RequestName']
except:
self.requestName = r[0].values()[0]['RequestName']
except Exception as ex:
msg = traceback.format_exc()
print "Exception during set up, reason: %s" % msg
raise ex
def __init__(self, url = 'http://localhost:5984', usePYCurl = False, ckey = None, cert = None, capath = None):
"""
Initialise requests
"""
JSONRequests.__init__(self, url, {"cachepath" : None, "pycurl" : usePYCurl, "key" : ckey, "cert" : cert, "capath" : capath})
self.accept_type = "application/json"
self["timeout"] = 600
def assignRequest(self,url,workflow,team,site,args={}):
params = {"action": "Assign",
"Team"+team: "checked",
"SiteWhitelist": [],
"SiteBlacklist": [],
"MergedLFNBase": "/store/user",
"UnmergedLFNBase": "/store/temp/user",
"MinMergeSize": 1,
"MaxMergeSize": 1,
"MaxMergeEvents": 50000,
#"AcquisitionEra": era,
"maxRSS": 4294967296,
"maxVSize": 4294967296,
"dashboard": "CMSYAATAnalysis",
"checkbox"+workflow: "checked"}
request = JSONRequests(url)
headers = {"Content-type": "application/x-www-form-urlencoded",
"Accept": "text/plain"}
request.post("/reqmgr/assign/handleAssignmentPage", params, headers)
(data, status, reason, _) = request.getresponse()
if status != 200:
self.debugHttpError(data, status, reason)
raise RuntimeError, "POST failed with code %s" % status
self.logger.info("Assigned the workflow %s" % workflow)
def __init__(self, url='http://localhost:5984', usePYCurl=True, ckey=None, cert=None, capath=None):
"""
Initialise requests
"""
JSONRequests.__init__(self, url,
{"cachepath": None, "pycurl": usePYCurl, "key": ckey, "cert": cert, "capath": capath})
self.accept_type = "application/json"
self["timeout"] = 600
def decodeJson(self, result):
"""
decodeJson
decode the response result reveiced from the server
"""
encoder = JSONRequests()
return encoder.decode(result)
def decodeJson(self, result):
"""
decodeJson
decode the response result reveiced from the server
"""
encoder = JSONRequests(idict={"pycurl" : True})
return encoder.decode(result)
def decodeJson(self, result):
"""
decodeJson
decode the response result reveiced from the server
"""
encoder = JSONRequests(idict={"pycurl": True})
return encoder.decode(result)
def decodeJson(self, result):
"""
decodeJson
decode the response result reveiced from the server
"""
encoder = JSONRequests()
return encoder.decode(result)
def wmbsServiceSetup(self, argstring, kargs={}, returnType="text"):
if returnType == "json":
request = JSONRequests(self.server_url)
else:
request = Requests(self.server_url)
results = request.get("/wmbs/%s/" % argstring, kargs)
return results
def wmbsServiceSetup(self, argstring, kargs={}, returnType='text'):
if returnType == 'json':
request = JSONRequests(self.server_url)
else:
request = Requests(self.server_url)
results = request.get("/wmbs/%s/" % argstring, kargs)
return results
def approveRequest(self, url,workflow):
params = {"requestName": workflow, "status": "assignment-approved"}
request = JSONRequests(url)
headers = {"Content-type": "application/x-www-form-urlencoded",
"Accept": "text/plain"}
request.put("/reqmgr/reqMgr/request", params, headers)
(data, status, reason, _) = request.getresponse()
if status != 200:
self.debugHttpError(data, status, reason)
raise RuntimeError, "PUT failed with code %s" % status
self.logger.info("Approved the workflow %s" % workflow)
def setUp(self):
"""
setUP global values
Database setUp is done in base class
"""
self.couchDBName = "reqmgr_t_0"
RESTBaseUnitTest.setUp(self)
self.testInit.setupCouch("%s" % self.couchDBName, "GroupUser",
"ConfigCache")
self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats")
reqMgrHost = self.config.getServerUrl()
self.jsonSender = JSONRequests(reqMgrHost)
def _verifyDBSCall(dbsURL, uri):
try:
# from WMCore.Services.DBS.DBS3Reader import DBS3Reader
# DBS3Reader(dbsURL).dbs.serverinfo()
from WMCore.Services.Requests import JSONRequests
jsonSender = JSONRequests(dbsURL)
result = jsonSender.get("/%s" % uri)
if not result[1] == 200:
raise WMSpecFactoryException("DBS is not connected: %s : %s" % (dbsURL, str(result)))
except:
raise WMSpecFactoryException("DBS is not responding: %s" % dbsURL)
return result[0]
def __init__(self, name, location, datasetpath):
"""
Just the necessary objects
Expects:
name: The blockname in full
location: The PNN of the site the block is at
"""
self.data = {
'dataset_conf_list': [], # List of dataset configurations
'file_conf_list': [], # List of files, the configuration for each
'files': [], # List of file objects
'block': {}, # Dict of block info
'processing_era': {}, # Dict of processing era info
'acquisition_era': {}, # Dict of acquisition era information
'primds': {}, # Dict of primary dataset info
'dataset': {}, # Dict of processed dataset info
'file_parent_list': [], # List of file parents
'dataset_parent_list':
[], # List of parent datasets (DBS requires this as list although it only allows one parent)
'close_settings': {}
} # Dict of info about block close settings
self.files = []
self.encoder = JSONRequests()
self.status = 'Open'
self.inBuff = False
self.startTime = time.time()
self.name = name
self.location = location
self.datasetpath = datasetpath
self.workflows = set()
self.data['block']['block_name'] = name
self.data['block']['origin_site_name'] = location
self.data['block']['open_for_writing'] = 1
self.data['block']['create_by'] = "WMAgent"
self.data['block']['creation_date'] = int(time.time())
self.data['block']['block_size'] = 0
self.data['block']['file_count'] = 0
self.data['block']['block_events'] = 0
self.data['close_settings'] = {}
self.data['close_settings']['block_close_max_wait_time'] = None
self.data['close_settings']['block_close_max_events'] = None
self.data['close_settings']['block_close_max_size'] = None
self.data['close_settings']['block_close_max_files'] = None
return
def makeRequest(self, url, params):
request = JSONRequests(url)
headers = {"Content-type": "application/x-www-form-urlencoded",
"Accept": "text/plain"}
request.post("/reqmgr/create/makeSchema", params, headers)
(data, status, reason, _) = request.getresponse()
if status != 303:
self.debugHttpError(data, status, reason)
raise RuntimeError, "POST failed with code %s" % status
workflow=data.split("'")[1].split('/')[-1]
self.logger.info('Injected workflow:',workflow,'into',url)
return workflow
class RequestManagerTest(RESTBaseUnitTest):
"""
Test RequestMgr Service client
It will start RequestMgr RESTService
Server DB is whatever env is set
This checks whether DS call makes without error and return the results.
This test only test service call returns without error.
The correctness of each function is tested in test/python/RequestManager_t/RequestMgr_t.py
"""
def initialize(self):
self.couchDBName = "reqmgr_t_0"
self.config = RequestManagerConfig(
'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel')
dbUrl = os.environ.get("DATABASE", None)
self.config.setDBUrl(dbUrl)
self.config.setFormatter('WMCore.WebTools.RESTFormatter')
self.config.setupRequestConfig()
self.config.setupCouchDatabase(dbName = self.couchDBName)
self.config.setPort(8888)
self.schemaModules = ["WMCore.RequestManager.RequestDB"]
def setUp(self):
RESTBaseUnitTest.setUp(self)
self.testInit.setupCouch("%s" % self.couchDBName,
"GroupUser", "ConfigCache")
self.params = {}
self.params['endpoint'] = self.config.getServerUrl()
self.reqService = RequestManagerDS(self.params)
self.jsonSender = JSONRequests(self.config.getServerUrl())
self.jsonSender.put('user/[email protected]')
self.jsonSender.put('group/PeopleLikeMe')
self.jsonSender.put('group/PeopleLikeMe/me')
self.jsonSender.put('version/CMSSW_3_5_8/slc5_ia32_gcc434')
schema = ReReco.getTestArguments()
schema['RequestName'] = 'TestReReco'
schema['RequestType'] = 'ReReco'
schema['CmsPath'] = "/uscmst1/prod/sw/cms"
schema['Requestor'] = '%s' % "me"
schema['Group'] = '%s' % "PeopleLikeMe"
schema['BlockWhitelist'] = ['/dataset/dataset/dataset#alpha']
schema['BlockBlacklist'] = ['/dataset/dataset/dataset#beta']
schema['Campaign'] = 'MyTestCampaign'
try:
r = self.jsonSender.put('request/' + schema['RequestName'], schema)
except Exception, ex:
print "Exception during set up, investigate exception instance attributes:"
print dir(ex)
return
self.requestName = r[0]['RequestName']
def setUp(self):
RESTBaseUnitTest.setUp(self)
self.testInit.setupCouch("%s" % self.couchDBName,
"GroupUser", "ConfigCache")
self.params = {}
self.params['endpoint'] = self.config.getServerUrl()
self.reqService = RequestManagerDS(self.params)
self.jsonSender = JSONRequests(self.config.getServerUrl())
self.jsonSender.put('user/[email protected]')
self.jsonSender.put('group/PeopleLikeMe')
self.jsonSender.put('group/PeopleLikeMe/me')
self.jsonSender.put('version/CMSSW_3_5_8/slc5_ia32_gcc434')
schema = ReReco.getTestArguments()
schema['RequestName'] = 'TestReReco'
schema['RequestType'] = 'ReReco'
schema['CmsPath'] = "/uscmst1/prod/sw/cms"
schema['Requestor'] = '%s' % "me"
schema['Group'] = '%s' % "PeopleLikeMe"
schema['BlockWhitelist'] = ['/dataset/dataset/dataset#alpha']
schema['BlockBlacklist'] = ['/dataset/dataset/dataset#beta']
schema['Campaign'] = 'MyTestCampaign'
try:
r = self.jsonSender.put('request/' + schema['RequestName'], schema)
except Exception, ex:
print "Exception during set up, investigate exception instance attributes:"
print dir(ex)
return
def makeRequest(self,
uri=None,
data=None,
type='GET',
incoming_headers={},
encode=True,
decode=True,
contentType=None,
cache=False):
"""
Make the request, handle any failed status, return just the data (for
compatibility). By default do not cache the response.
TODO: set caching in the calling methods.
"""
try:
if not cache:
incoming_headers.update({'Cache-Control': 'no-cache'})
result, status, reason, cached = JSONRequests.makeRequest(
self, uri, data, type, incoming_headers, encode, decode,
contentType)
except HTTPException as e:
self.checkForCouchError(getattr(e, "status", None),
getattr(e, "reason", None), data)
return result
def setUp(self):
RESTBaseUnitTest.setUp(self)
self.testInit.setupCouch("%s" % self.couchDBName, "GroupUser", "ConfigCache", "ReqMgr")
self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats")
# logging stuff from TestInit is broken, setting myself
l = logging.getLogger()
l.setLevel(logging.DEBUG)
self.params = {}
self.params['endpoint'] = self.config.getServerUrl()
self.reqService = RequestManagerDS(self.params)
self.jsonSender = JSONRequests(self.config.getServerUrl())
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
schema['ConfigCacheID'] = self.createConfig()
schema['CouchDBName'] = self.couchDBName
schema['CouchWorkloadDBName'] = self.couchDBName
try:
r = self.jsonSender.put('request', schema)
try:
self.requestName = r[0]['RequestName']
except:
self.requestName = r[0].values()[0]['RequestName']
except Exception as ex:
msg = traceback.format_exc()
print("Exception during set up, reason: %s" % msg)
raise ex
def __init__(self, dict):
defaultdict = {'endpoint': "https://cmsweb.cern.ch/registration/",
'cacheduration': 1,
'cachepath': '/tmp'}
defaultdict.update(dict)
defaultdict["method"] = 'PUT'
if 'logger' not in defaultdict.keys():
logging.basicConfig(level = logging.DEBUG,
format = '%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
datefmt = '%m-%d %H:%M',
filename = defaultdict['cachepath'] + '/regsvc.log',
filemode = 'w')
defaultdict['logger'] = logging.getLogger('RegService')
Service.__init__(self, defaultdict)
JSONRequests.__init__(self, defaultdict['endpoint'])
def testPush(self):
reg_info ={
"location": "https://globaldbs",
"admin": "*****@*****.**",
"type": "DBS",
"name": "Global DBS",
"timeout": 2
}
reg = Registration({'server': self.testInit.couchUrl, 'database': self.couch_db}, reg_info)
json = BasicAuthJSONRequests(self.testInit.couchUrl)
data = json.get('/%s/%s' % (self.couch_db, str(reg_info['location'].__hash__())))
for k, v in reg_info.items():
if k != 'timestamp':
assert data[0][k] == v, \
"Registration incomplete: %s should equal %s for key %s" % (data[0][k], v, k)
def setUp(self):
"""
setUP global values
Database setUp is done in base class
"""
self.couchDBName = "reqmgr_t_0"
RESTBaseUnitTest.setUp(self)
self.testInit.setupCouch("%s" % self.couchDBName, "ConfigCache", "ReqMgr")
self.testInit.setupCouch("%s_wmstats" % self.couchDBName,
"WMStats")
self.testInit.setupCouch("%s_acdc" % self.couchDBName,
"ACDC", "GroupUser")
reqMgrHost = self.config.getServerUrl()
self.jsonSender = JSONRequests(reqMgrHost)
self.params = {}
self.params['endpoint'] = reqMgrHost
self.reqService = RequestManager(self.params)
def __init__(self, dict):
defaultdict = {
'endpoint': "https://cmsweb.cern.ch/registration/",
'cacheduration': 1,
'cachepath': '/tmp'
}
defaultdict.update(dict)
defaultdict["method"] = 'PUT'
if 'logger' not in defaultdict.keys():
logging.basicConfig(
level=logging.DEBUG,
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
datefmt='%m-%d %H:%M',
filename=defaultdict['cachepath'] + '/regsvc.log',
filemode='w')
defaultdict['logger'] = logging.getLogger('RegService')
Service.__init__(self, defaultdict)
JSONRequests.__init__(self, defaultdict['endpoint'])
def makeRequest(self, uri = None, data = None, verb = 'GET'):
"""
Make a request to the remote database. for a give URI. The type of
request will determine the action take by the server (be careful with
DELETE!). Data should be a dictionary of {dataname: datavalue}.
Returns a tuple of the data from the server, decoded using the
appropriate method the response status and the response reason, to be
used in error handling.
You can override the method to encode/decode your data by passing in an
encoding/decoding function to this method. Your encoded data must end up
as a string.
"""
data = data or {}
headers = {
"User-agent": "CRABClient/%s" % self['version'],
"Accept": "*/*",
}
#Quoting the uri since it can contain the request name, and therefore spaces (see #2557)
uri = urllib.quote(uri)
caCertPath = self.getCACertPath()
url = 'https://' + self['host'] + uri
#retries this up to self['retry'] times a range of exit codes
for i in range(self['retry'] + 1):
try:
response, datares = self['conn'].request(url, data, encode=True, headers=headers, verb=verb, doseq = True,
ckey=self['key'], cert=self['cert'], capath=caCertPath,
verbose=self['verbose'])
except Exception as ex:
#add here other temporary errors we need to retry
if (not retriableError(ex)) or (i == self['retry']):
msg = "Fatal error trying to connect to %s using %s" % (url, data)
self.logger.error(msg)
raise #really exit and raise exception if this was the last retry or the exit code is not among the list of the one we retry
sleeptime = 20 * (i + 1)
msg = "Sleeping %s seconds after HTTP error. Error details: " % sleeptime
if hasattr(ex, 'headers'):
msg += str(ex.headers)
else:
msg += str(ex)
self.logger.debug(msg)
time.sleep(sleeptime) #sleeps 20s the first time, 40s the second time and so on
else:
break
try:
result = JSONRequests(idict={"pycurl" : True}).decode(datares)
except Exception as ex:
msg = "Fatal error reading data from %s using %s" % (url, data)
self.logger.error(msg)
raise #really exit and raise exception
return result, response.status, response.reason
def setUp(self):
"""
setUP global values
Database setUp is done in base class
"""
self.couchDBName = "reqmgr_t_0"
RESTBaseUnitTest.setUp(self)
self.testInit.setupCouch("%s" % self.couchDBName, "GroupUser", "ConfigCache")
self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats")
reqMgrHost = self.config.getServerUrl()
self.jsonSender = JSONRequests(reqMgrHost)
def __init__(self, cfg, statedir, testName):
self.server = RESTMain(cfg, statedir)
self.testName = testName
self.cofig = cfg
self.port = cfg.main.port
self.host = '127.0.0.1'
self.serverUrl = "http://%s:%s/%s/" % (self.host, self.port, cfg.main.application)
## test permission
#test_authz_key = fake_authz_key_file()
#self.header = fake_authz_headers(test_authz_key.data, roles = {"Global Admin": {'group': ['global']}})
self.jsonSender = JSONRequests(self.serverUrl)
def setUp(self):
"""
setUP global values
Database setUp is done in base class
"""
self.couchDBName = "reqmgr_t_0"
RESTBaseUnitTest.setUp(self, initRoot=False)
self.testInit.setupCouch("%s" % self.couchDBName, "ConfigCache",
"ReqMgr")
reqMgrHost = self.config.getServerUrl()
self.jsonSender = JSONRequests(reqMgrHost)
def __init__(self, name, location, das):
"""
Just the necessary objects
Expects:
name: The blockname in full
location: The SE-name of the site the block is at
"""
self.data = {
'dataset_conf_list': [], # List of dataset configurations
'file_conf_list': [], # List of files, the configuration for each
'files': [], # List of file objects
'block': {}, # Dict of block info
'processing_era': {}, # Dict of processing era info
'acquisition_era': {}, # Dict of acquisition era information
'primds': {}, # Dict of primary dataset info
'dataset': {}, # Dict of processed dataset info
'file_parent_list': []
} # List of file parents
self.files = []
self.encoder = JSONRequests()
self.status = 'Open'
self.inBuff = False
self.startTime = time.time()
self.name = name
self.location = location
self.das = das
self.data['block']['block_name'] = name
self.data['block']['origin_site_name'] = location
self.data['block'][
'open_for_writing'] = 0 # If we're sending a block, it better be open
self.data['block']['create_by'] = "WMAgent"
self.data['block']['creation_date'] = int(time.time())
self.data['block']['block_size'] = 0
self.data['block']['file_count'] = 0
return
def testPush(self):
reg_info = {
"location": "https://globaldbs",
"admin": "*****@*****.**",
"type": "DBS",
"name": "Global DBS",
"timeout": 2
}
reg = Registration(
{
'server': self.testInit.couchUrl,
'database': self.couch_db
}, reg_info)
json = BasicAuthJSONRequests(self.testInit.couchUrl)
data = json.get('/%s/%s' %
(self.couch_db, str(reg_info['location'].__hash__())))
for k, v in reg_info.items():
if k != 'timestamp':
assert data[0][k] == v, \
"Registration incomplete: %s should equal %s for key %s" % (data[0][k], v, k)
def __init__(self, slaveClassName, totalSlaves, componentDir,
config, slaveInit = None, namespace = None):
"""
__init__
Constructor for the process pool. The slave class name must be based
inside the WMComponent namespace. For examples, the JobAccountant would
pass in 'JobAccountant.AccountantWorker' to run the AccountantWorker
class. All log files will be stored in the component directory that is
passed in. Each slave will have its own log file.
Note that the config is only used to determine database connection
parameters. It is not passed to the slave class. The slaveInit
parameter will be serialized and passed to the slave class's
constructor.
"""
self.enqueueIndex = 0
self.dequeueIndex = 0
self.runningWork = 0
#Use the Services.Requests JSONizer, which handles __to_json__ calls
self.jsonHandler = JSONRequests()
# heartbeat should be registered at this point
if getattr(config.Agent, "useHeartbeat", True):
self.heartbeatAPI = HeartbeatAPI(getattr(config.Agent, "componentName", "ProcPoolSlave"))
self.slaveClassName = slaveClassName
self.componentDir = componentDir
self.config = config
# Grab the python version from the current version
# Assume naming convention pythonA.B, i.e., python2.4 for v2.4.X
majorVersion = sys.version_info[0]
minorVersion = sys.version_info[1]
if majorVersion and minorVersion:
self.versionString = "python%i.%i" % (majorVersion, minorVersion)
else:
self.versionString = "python2.4"
self.workers = []
self.nSlaves = totalSlaves
self.slaveInit = slaveInit
self.namespace = namespace
# Now actually create the slaves
self.createSlaves()
return
class RequestManagerTest(RESTBaseUnitTest):
"""
Test RequestMgr Service client
It will start RequestMgr RESTService
Server DB is whatever env is set
This checks whether DS call makes without error and return the results.
This test only test service call returns without error.
The correctness of each function is tested in test/python/RequestManager_t/RequestMgr_t.py
"""
def initialize(self):
self.couchDBName = "reqmgr_t_0"
self.config = RequestManagerConfig(
'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel')
dbUrl = os.environ.get("DATABASE", None)
self.config.setDBUrl(dbUrl)
self.config.setFormatter('WMCore.WebTools.RESTFormatter')
self.config.setupRequestConfig()
self.config.setupCouchDatabase(dbName=self.couchDBName)
self.config.setPort(8888)
self.schemaModules = ["WMCore.RequestManager.RequestDB"]
def setUp(self):
RESTBaseUnitTest.setUp(self)
self.testInit.setupCouch("%s" % self.couchDBName, "GroupUser",
"ConfigCache", "ReqMgr")
self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats")
# logging stuff from TestInit is broken, setting myself
l = logging.getLogger()
l.setLevel(logging.DEBUG)
self.params = {}
self.params['endpoint'] = self.config.getServerUrl()
self.reqService = RequestManagerDS(self.params)
self.jsonSender = JSONRequests(self.config.getServerUrl())
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName=userName,
groupName=groupName,
teamName=teamName)
schema['ConfigCacheID'] = self.createConfig()
schema['CouchDBName'] = self.couchDBName
try:
r = self.jsonSender.put('request/' + schema['RequestName'], schema)
self.requestName = r[0]['RequestName']
except Exception, ex:
print "Exception during set up, reason: %s" % ex
raise ex
def __init__(self, cfg, statedir, testName):
self.server = RESTMain(cfg, statedir)
self.testName = testName
self.config = cfg
self.port = cfg.main.port
self.host = '127.0.0.1'
self.serverUrl = "http://%s:%s/%s/" % (self.host, self.port, cfg.main.application)
## test authentication using fake filepermission
self.test_authz_key = fake_authz_key_file(False)
self.config.main.tools.cms_auth.key_file = self.test_authz_key.name
#self.header = fake_authz_headers(test_authz_key.data, roles = {"Admin": {'group': ['ReqMgr']}})
self.jsonSender = JSONRequests(self.serverUrl)
def getRequestInfo():
requestName = sys.argv[1]
print "Getting information from cmsweb about %s... %s" % (
requestName, time.strftime('%H:%M:%S'))
requestor = JSONRequests(url='https://cmsweb.cern.ch/reqmgr/reqMgr')
response = requestor.get('/request?requestName=%s' % requestName)
if response[1] != 200:
raise RuntimeError("Request information was not available!")
requestDict = response[0]
#print requestDict.get('RunWhitelist')
#print requestDict.get('RunBlacklist')
if not requestDict.get('RunWhitelist'):
requestDict['RunWhitelist'] = []
if not requestDict.get('RunBlacklist'):
requestDict['RunBlacklist'] = []
if not requestDict.get('BlockWhitelist'):
requestDict['BlockWhitelist'] = []
if not requestDict.get('BlockBlacklist'):
requestDict['BlockBlacklist'] = []
compactRequest = {
'InputDataset': requestDict['InputDataset'],
'RunWhitelist': requestDict['RunWhitelist'],
'RunBlacklist': requestDict['RunBlacklist'],
'BlockWhitelist': requestDict['BlockWhitelist'],
'BlockBlacklist': requestDict['BlockBlacklist'],
'OutputDatasets': []
}
response = requestor.get('/outputDatasetsByRequestName?requestName=%s' %
requestName)
if response[1] != 200:
raise RuntimeError("Output dataset information was not available!")
compactRequest['OutputDatasets'] = response[0]
print "Done querying cmsweb... %s" % time.strftime('%H:%M:%S')
return compactRequest
class RequestManagerTest(RESTBaseUnitTest):
"""
Test RequestMgr Service client
It will start RequestMgr RESTService
Server DB is whatever env is set
This checks whether DS call makes without error and return the results.
This test only test service call returns without error.
The correctness of each function is tested in test/python/RequestManager_t/RequestMgr_t.py
"""
def initialize(self):
self.couchDBName = "reqmgr_t_0"
self.config = RequestManagerConfig(
'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel')
dbUrl = os.environ.get("DATABASE", None)
self.config.setDBUrl(dbUrl)
self.config.setFormatter('WMCore.WebTools.RESTFormatter')
self.config.setupRequestConfig()
self.config.setupCouchDatabase(dbName = self.couchDBName)
self.config.setPort(8888)
self.schemaModules = ["WMCore.RequestManager.RequestDB"]
def setUp(self):
RESTBaseUnitTest.setUp(self)
self.testInit.setupCouch("%s" % self.couchDBName, "GroupUser", "ConfigCache")
self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats")
# logging stuff from TestInit is broken, setting myself
l = logging.getLogger()
l.setLevel(logging.DEBUG)
self.params = {}
self.params['endpoint'] = self.config.getServerUrl()
self.reqService = RequestManagerDS(self.params)
self.jsonSender = JSONRequests(self.config.getServerUrl())
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
try:
r = self.jsonSender.put('request/' + schema['RequestName'], schema)
except Exception, ex:
print "Exception during set up, reason: %s" % ex
return
self.requestName = r[0]['RequestName']
def setUp(self):
RESTBaseUnitTest.setUp(self)
self.testInit.setupCouch("%s" % self.couchDBName, "GroupUser", "ConfigCache")
self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats")
# logging stuff from TestInit is broken, setting myself
l = logging.getLogger()
l.setLevel(logging.DEBUG)
self.params = {}
self.params['endpoint'] = self.config.getServerUrl()
self.reqService = RequestManagerDS(self.params)
self.jsonSender = JSONRequests(self.config.getServerUrl())
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
try:
r = self.jsonSender.put('request/' + schema['RequestName'], schema)
except Exception, ex:
print "Exception during set up, reason: %s" % ex
return
def __init__(self, name, location, datasetpath):
"""
Just the necessary objects
Expects:
name: The blockname in full
location: The PNN of the site the block is at
"""
self.data = {'dataset_conf_list': [], # List of dataset configurations
'file_conf_list': [], # List of files, the configuration for each
'files': [], # List of file objects
'block': {}, # Dict of block info
'processing_era': {}, # Dict of processing era info
'acquisition_era': {}, # Dict of acquisition era information
'primds': {}, # Dict of primary dataset info
'dataset': {}, # Dict of processed dataset info
'file_parent_list': [], # List of file parents
'dataset_parent_list': [], # List of parent datasets (DBS requires this as list although it only allows one parent)
'close_settings': {}} # Dict of info about block close settings
self.files = []
self.encoder = JSONRequests()
self.status = 'Open'
self.inBuff = False
self.startTime = time.time()
self.name = name
self.location = location
self.datasetpath = datasetpath
self.workflows = set()
self.data['block']['block_name'] = name
self.data['block']['origin_site_name'] = location
self.data['block']['open_for_writing'] = 1
self.data['block']['create_by'] = "WMAgent"
self.data['block']['creation_date'] = int(time.time())
self.data['block']['block_size'] = 0
self.data['block']['file_count'] = 0
self.data['block']['block_events'] = 0
self.data['close_settings'] = {}
self.data['close_settings']['block_close_max_wait_time'] = None
self.data['close_settings']['block_close_max_events'] = None
self.data['close_settings']['block_close_max_size'] = None
self.data['close_settings']['block_close_max_files'] = None
return
def setUp(self):
"""
setUP global values
"""
RESTBaseUnitTest.setUp(self)
self.params = {}
self.params['endpoint'] = self.config.getServerUrl()
self.reqService = RequestManagerDS(self.params)
self.jsonSender = JSONRequests(self.config.getServerUrl())
self.requestSchema = getRequestSchema()
self.jsonSender.put('group/PeopleLikeMe')
self.jsonSender.put('user/[email protected]')
self.jsonSender.put('group/PeopleLikeMe/me')
self.jsonSender.put('version/CMSSW_3_5_8')
self.jsonSender.put('request/' + self.requestSchema['RequestName'],
self.requestSchema)
def makeRequest(self, uri=None, data=None, type='GET', incoming_headers = {},
encode=True, decode=True, contentType=None, cache=False):
"""
Make the request, handle any failed status, return just the data (for
compatibility). By default do not cache the response.
TODO: set caching in the calling methods.
"""
try:
if not cache:
incoming_headers.update({'Cache-Control':'no-cache'})
result, status, reason, cached = JSONRequests.makeRequest(
self, uri, data, type, incoming_headers,
encode, decode,contentType)
except HTTPException, e:
self.checkForCouchError(getattr(e, "status", None),
getattr(e, "reason", None), data)
def setUp(self):
"""
setUP global values
Database setUp is done in base class
"""
self.couchDBName = "reqmgr_t_0"
RESTBaseUnitTest.setUp(self)
self.testInit.setupCouch("%s" % self.couchDBName, "ConfigCache", "ReqMgr")
self.testInit.setupCouch("%s_wmstats" % self.couchDBName,
"WMStats")
self.testInit.setupCouch("%s_acdc" % self.couchDBName,
"ACDC", "GroupUser")
reqMgrHost = self.config.getServerUrl()
self.jsonSender = JSONRequests(reqMgrHost)
self.params = {}
self.params['endpoint'] = reqMgrHost
self.reqService = RequestManager(self.params)
def setUp(self):
"""
setUP global values
"""
RESTBaseUnitTest.setUp(self)
reqMgrHost = self.config.getServerUrl()
self.requestSchema = getRequestSchema()
print reqMgrHost
self.jsonSender = JSONRequests(reqMgrHost)
#self.requestTypes = ['ReReco', 'StoreResults', 'CmsGen', 'Reco']
#self.requestTypes = ['ReReco', 'MonteCarlo']
self.requestTypes = ['ReReco']
if 'me' in self.jsonSender.get('user')[0]:
self.jsonSender.delete('user/me')
self.assertFalse('me' in self.jsonSender.get('user')[0])
self.assertEqual(self.jsonSender.put('user/[email protected]')[1], 200)
self.assertTrue('me' in self.jsonSender.get('user')[0])
if 'PeopleLikeMe' in self.jsonSender.get('group')[0]:
self.jsonSender.delete('group/PeopleLikeMe')
self.assertFalse('PeopleLikeMe' in self.jsonSender.get('group')[0])
self.assertEqual(self.jsonSender.put('group/PeopleLikeMe')[1], 200)
self.assertTrue( 'PeopleLikeMe' in self.jsonSender.get('group')[0])
self.jsonSender.put('group/PeopleLikeMe/me')
users = json.loads(self.jsonSender.get('group/PeopleLikeMe')[0])['users']
self.assertTrue('me' in users)
groups = json.loads(self.jsonSender.get('user/me')[0])['groups']
self.assertTrue('PeopleLikeMe' in groups)
groups2 = self.jsonSender.get('group?user=me')[0]
self.assertTrue('PeopleLikeMe' in groups2)
if 'White Sox' in self.jsonSender.get('team')[0]:
self.jsonSender.delete(urllib.quote('team/White Sox'))
self.assertFalse('White Sox' in self.jsonSender.get('team')[0])
self.assertEqual(self.jsonSender.put(urllib.quote('team/White Sox'))[1], 200)
self.assertTrue('White Sox' in self.jsonSender.get('team')[0])
# some foreign key stuff to dealwith
#self.assertFalse('CMSSW_X_Y_Z' in self.jsonSender.get('version')[0])
self.assertTrue(self.jsonSender.put('version/CMSSW_3_5_8')[1] == 200)
self.assertTrue('CMSSW_3_5_8' in self.jsonSender.get('version')[0])
def __init__(self, name, location, das):
"""
Just the necessary objects
Expects:
name: The blockname in full
location: The SE-name of the site the block is at
"""
self.data = {'dataset_conf_list': [], # List of dataset configurations
'file_conf_list': [], # List of files, the configuration for each
'files': [], # List of file objects
'block': {}, # Dict of block info
'processing_era': {}, # Dict of processing era info
'acquisition_era': {}, # Dict of acquisition era information
'primds': {}, # Dict of primary dataset info
'dataset': {}, # Dict of processed dataset info
'file_parent_list': []} # List of file parents
self.files = []
self.encoder = JSONRequests()
self.status = 'Open'
self.inBuff = False
self.startTime = time.time()
self.name = name
self.location = location
self.das = das
self.data['block']['block_name'] = name
self.data['block']['origin_site_name'] = location
self.data['block']['open_for_writing'] = 0 # If we're sending a block, it better be open
self.data['block']['create_by'] = "WMAgent"
self.data['block']['creation_date'] = int(time.time())
self.data['block']['block_size'] = 0
self.data['block']['file_count'] = 0
return
def validate_input(self, input, verb, method):
assert 'type' in input.keys(
), "no type provided - what kind of plot do you want?"
if method == 'doc':
return input
valid_data = {}
assert 'data' in input.keys() or 'url' in input.keys(
), "neither data nor url provided - please provide at least one"
if 'url' in input.keys():
match = URL_REGEX.match(input['url'])
assert match != None, "`%s' is not a valid URL" % input['url']
host = match.group(1) + '://' + match.group(3)
if match.group(4):
host += match.group(4)
uri = match.group(5)
result, status, reason, fromcache = JSONRequests(host).get(uri)
valid_data.update(result)
if 'data' in input.keys():
valid_data.update(json.loads(urllib.unquote(input['data'])))
return {'data': valid_data, 'type': input['type']}
class ReqMgrPriorityTest(RESTBaseUnitTest):
"""
_ReqMgrPriorityTest_
Basic test for setting the priority in ReqMgr Services
"""
def setUp(self):
"""
setUP global values
Database setUp is done in base class
"""
self.couchDBName = "reqmgr_t_0"
RESTBaseUnitTest.setUp(self)
self.testInit.setupCouch("%s" % self.couchDBName,
"GroupUser", "ConfigCache")
reqMgrHost = self.config.getServerUrl()
self.jsonSender = JSONRequests(reqMgrHost)
return
def initialize(self):
self.config = RequestManagerConfig(
'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel')
self.config.setFormatter('WMCore.WebTools.RESTFormatter')
self.config.setupRequestConfig()
self.config.setupCouchDatabase(dbName = self.couchDBName)
self.config.setPort(8888)
self.schemaModules = ["WMCore.RequestManager.RequestDB"]
return
def tearDown(self):
"""
_tearDown_
Basic tear down of database
"""
RESTBaseUnitTest.tearDown(self)
self.testInit.tearDownCouch()
return
def setupSchema(self, groupName = 'PeopleLikeMe',
userName = '******', teamName = 'White Sox',
CMSSWVersion = 'CMSSW_3_5_8',
scramArch = 'slc5_ia32_gcc434'):
"""
_setupSchema_
Set up a test schema so that we can run a test request.
Standardization!
"""
self.jsonSender.put('user/%[email protected]' % userName)
self.jsonSender.put('group/%s' % groupName)
self.jsonSender.put('group/%s/%s' % (groupName, userName))
self.jsonSender.put(urllib.quote('team/%s' % teamName))
self.jsonSender.put('version/%s/%s' % (CMSSWVersion, scramArch))
schema = ReReco.getTestArguments()
schema['RequestName'] = 'TestReReco'
schema['RequestType'] = 'ReReco'
schema['CmsPath'] = "/uscmst1/prod/sw/cms"
schema['Requestor'] = '%s' % userName
schema['Group'] = '%s' % groupName
return schema
def loadWorkload(self, requestName):
"""
_loadWorkload_
Load the workload from couch after we've saved it there.
"""
workload = WMWorkloadHelper()
url = '%s/%s/%s/spec' % (os.environ['COUCHURL'], self.couchDBName,
requestName)
workload.load(url)
return workload
def changeStatusAndCheck(self, requestName, statusName):
"""
_changeStatusAndCheck_
Change the status of a request and make sure that
the request actually did it.
"""
self.jsonSender.put('request/%s?status=%s' % (requestName, statusName))
result = self.jsonSender.get('request/%s' % requestName)
self.assertEqual(result[0]['RequestStatus'], statusName)
return
def testA_RequestPriority(self):
"""
_priorityChanges_
Do some fairly standard priority changes to the Request and
see how things react
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
CMSSWVersion = 'CMSSW_3_5_8'
schema = self.setupSchema(userName = userName,
groupName = groupName,
teamName = teamName,
CMSSWVersion = CMSSWVersion)
result = self.jsonSender.put('request/testRequest', schema)
self.assertEqual(result[1], 200)
requestName = result[0]['RequestName']
workload = self.loadWorkload(requestName = requestName)
self.assertEqual(workload.priority(), 0)
# Reset user, group priorities to 0
self.jsonSender.post('user/%s?priority=0' % userName)
self.jsonSender.post('group/%s?priority=0' % groupName)
# Set priority == 5
priority = 5
self.jsonSender.put('request/%s?priority=%s' % (requestName, priority))
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(request['ReqMgrRequestBasePriority'], priority)
workload = self.loadWorkload(requestName = requestName)
self.assertEqual(workload.priority(), priority)
# Set priority == 100
priority = 100
self.jsonSender.put('request/%s?priority=%s' % (requestName, priority))
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(request['ReqMgrRequestBasePriority'], priority)
workload = self.loadWorkload(requestName = requestName)
self.assertEqual(workload.priority(), priority)
# Set priority == -1
priority = -1
self.jsonSender.put('request/%s?priority=%s' % (requestName, priority))
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(request['ReqMgrRequestBasePriority'], priority)
workload = self.loadWorkload(requestName = requestName)
self.assertEqual(workload.priority(), priority)
# Let's move the request around a bit
self.changeStatusAndCheck(requestName = requestName,
statusName = 'testing-approved')
self.changeStatusAndCheck(requestName = requestName,
statusName = 'testing')
self.changeStatusAndCheck(requestName = requestName,
statusName = 'tested')
self.changeStatusAndCheck(requestName = requestName,
statusName = 'assignment-approved')
self.jsonSender.put(urllib.quote('assignment/%s/%s' % (teamName, requestName)))
# Set priority == 99
priority = 99
self.jsonSender.put('request/%s?priority=%s' % (requestName, priority))
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(request['ReqMgrRequestBasePriority'], priority)
workload = self.loadWorkload(requestName = requestName)
self.assertEqual(workload.priority(), priority)
return
def testB_InvalidPriority(self):
"""
_InvalidPriority_
Put in a bunch of invalid values for priorities and
see what the code makes of them.
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
CMSSWVersion = 'CMSSW_3_5_8'
schema = self.setupSchema(userName = userName,
groupName = groupName,
teamName = teamName,
CMSSWVersion = CMSSWVersion)
result = self.jsonSender.put('request/testRequest', schema)
self.assertEqual(result[1], 200)
requestName = result[0]['RequestName']
raises = False
try:
priority = sys.maxint + 1
self.jsonSender.put('request/%s?priority=%s' % (requestName, priority))
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue("Priority must have abs() less then MAXINT!" in ex.result)
self.assertTrue(raises)
raises = False
try:
priority = -1 - sys.maxint
self.jsonSender.put('request/%s?priority=%s' % (requestName, priority))
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
print ex.result
self.assertTrue("Priority must have abs() less then MAXINT!" in ex.result)
class ReqMgrTest(RESTBaseUnitTest):
"""
_ReqMgrTest_
Basic test for the ReqMgr services.
Setup is done off-screen in RESTBaseUnitTest - this makes
things confusing
"""
def setUp(self):
"""
setUP global values
Database setUp is done in base class
"""
self.couchDBName = "reqmgr_t_0"
RESTBaseUnitTest.setUp(self)
self.testInit.setupCouch("%s" % self.couchDBName, "GroupUser", "ConfigCache")
self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats")
reqMgrHost = self.config.getServerUrl()
self.jsonSender = JSONRequests(reqMgrHost)
return
def initialize(self):
self.config = RequestManagerConfig("WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel")
self.config.setFormatter("WMCore.WebTools.RESTFormatter")
self.config.setupRequestConfig()
self.config.setupCouchDatabase(dbName=self.couchDBName)
self.config.setPort(12888)
self.schemaModules = ["WMCore.RequestManager.RequestDB"]
return
def tearDown(self):
"""
tearDown
Tear down everything
"""
RESTBaseUnitTest.tearDown(self)
self.testInit.tearDownCouch()
return
def setupSchema(
self,
groupName="PeopleLikeMe",
userName="******",
teamName="White Sox",
CMSSWVersion="CMSSW_3_5_8",
scramArch="slc5_ia32_gcc434",
):
"""
_setupSchema_
Set up a test schema so that we can run a test request.
Standardization!
"""
self.jsonSender.put("user/%[email protected]" % userName)
self.jsonSender.put("group/%s" % groupName)
self.jsonSender.put("group/%s/%s" % (groupName, userName))
self.jsonSender.put(urllib.quote("team/%s" % teamName))
self.jsonSender.put("version/%s/%s" % (CMSSWVersion, scramArch))
schema = ReReco.getTestArguments()
schema["RequestName"] = "TestReReco"
schema["RequestType"] = "ReReco"
schema["CmsPath"] = "/uscmst1/prod/sw/cms"
schema["Requestor"] = "%s" % userName
schema["Group"] = "%s" % groupName
schema["CustodialSite"] = "US_T1_FNAL"
schema["TimePerEvent"] = "12"
schema["Memory"] = 3000
schema["SizePerEvent"] = 512
return schema
def createConfig(self, bad=False):
"""
_createConfig_
Create a config of some sort that we can load out of ConfigCache
"""
PSetTweak = {
"process": {
"outputModules_": ["ThisIsAName"],
"ThisIsAName": {"dataset": {"dataTier": "RECO", "filterName": "Filter"}},
}
}
BadTweak = {
"process": {
"outputModules_": ["ThisIsAName1", "ThisIsAName2"],
"ThisIsAName1": {"dataset": {"dataTier": "RECO", "filterName": "Filter"}},
"ThisIsAName2": {"dataset": {"dataTier": "RECO", "filterName": "Filter"}},
}
}
configCache = ConfigCache(os.environ["COUCHURL"], couchDBName=self.couchDBName)
configCache.createUserGroup(groupname="testGroup", username="******")
if bad:
configCache.setPSetTweaks(PSetTweak=BadTweak)
else:
configCache.setPSetTweaks(PSetTweak=PSetTweak)
configCache.save()
return configCache.getCouchID()
@attr("integration")
def testA_testBasicSetUp(self):
"""
_testBasicSetUp_
Moving the tests that were in the setUp category out of it,
mostly because I want to make sure that they don't fail inside
the setUp statement.
"""
if "me" in self.jsonSender.get("user")[0]:
self.jsonSender.delete("user/me")
self.assertFalse("me" in self.jsonSender.get("user")[0])
self.assertEqual(self.jsonSender.put("user/[email protected]")[1], 200)
self.assertTrue("me" in self.jsonSender.get("user")[0])
if "PeopleLikeMe" in self.jsonSender.get("group")[0]:
self.jsonSender.delete("group/PeopleLikeMe")
self.assertFalse("PeopleLikeMe" in self.jsonSender.get("group")[0])
self.assertEqual(self.jsonSender.put("group/PeopleLikeMe")[1], 200)
self.assertTrue("PeopleLikeMe" in self.jsonSender.get("group")[0])
self.jsonSender.put("group/PeopleLikeMe/me")
users = self.jsonSender.get("group/PeopleLikeMe")[0]["users"]
self.assertTrue("me" in users)
groups = self.jsonSender.get("user/me")[0]["groups"]
self.assertTrue("PeopleLikeMe" in groups)
groups2 = self.jsonSender.get("group?user=me")[0]
self.assertTrue("PeopleLikeMe" in groups2)
if "White Sox" in self.jsonSender.get("team")[0]:
self.jsonSender.delete(urllib.quote("team/White Sox"))
self.assertFalse("White Sox" in self.jsonSender.get("team")[0])
self.assertEqual(self.jsonSender.put(urllib.quote("team/White Sox"))[1], 200)
self.assertTrue("White Sox" in self.jsonSender.get("team")[0])
# some foreign key stuff to dealwith
self.assertTrue(self.jsonSender.put("version/CMSSW_3_5_8")[1] == 200)
self.assertTrue("CMSSW_3_5_8" in self.jsonSender.get("version")[0])
return
@attr("integration")
def testB_ReReco(self):
"""
_ReReco_
Try a basic ReReco workflow
"""
schema = self.setupSchema()
schema["RequestNumEvents"] = 100
schema["RequestEventSize"] = 101
self.doRequest(schema)
return
def doRequest(self, schema):
"""
_doRequest_
Run all tests on a basic ReReco workflow
"""
requestName = schema["RequestName"]
self.assertRaises(HTTPException, self.jsonSender.delete, "request/%s" % requestName)
result = self.jsonSender.put("request/%s" % (requestName), schema)
self.assertEqual(result[1], 200)
requestName = result[0]["RequestName"]
self.assertEqual(self.jsonSender.get("request/%s" % requestName)[0]["RequestName"], requestName)
self.jsonSender.put("request/%s?status=assignment-approved" % requestName)
me = self.jsonSender.get("user/me")[0]
self.assertTrue(requestName in me["requests"])
self.assertEqual(self.jsonSender.put("request/%s?priority=5" % requestName)[1], 200)
self.assertEqual(self.jsonSender.post("user/me?priority=6")[1], 200)
self.assertEqual(self.jsonSender.post("group/PeopleLikeMe?priority=7")[1], 200)
# default priority of group and user of 1
request = self.jsonSender.get("request/%s" % requestName)[0]
self.assertEqual(request["ReqMgrRequestBasePriority"], 5)
self.assertEqual(request["ReqMgrRequestorBasePriority"], 6)
self.assertEqual(request["ReqMgrGroupBasePriority"], 7)
self.assertEqual(request["RequestPriority"], 5 + 6 + 7)
# Check LFN Bases
self.assertEqual(request["UnmergedLFNBase"], "/store/unmerged")
self.assertEqual(request["MergedLFNBase"], "/store/data")
# Check random other
self.assertEqual(request["CustodialSite"], "US_T1_FNAL")
# Check Num events
self.assertEqual(request["RequestNumEvents"], 100)
self.assertEqual(request["RequestEventSize"], 101)
# only certain transitions allowed
# self.assertEqual(self.jsonSender.put('request/%s?status=running' % requestName)[1], 400)
self.assertRaises(HTTPException, self.jsonSender.put, "request/%s?status=running" % requestName)
request = self.jsonSender.get("request/%s" % requestName)[0]
self.assertEqual(request["RequestStatus"], "assignment-approved")
self.assertTrue(self.jsonSender.put(urllib.quote("assignment/White Sox/%s" % requestName))[1] == 200)
requestsAndSpecs = self.jsonSender.get(urllib.quote("assignment/White Sox"))[0]
self.assertTrue(requestName in requestsAndSpecs[0])
workloadHelper = WMWorkloadHelper()
workloadHelper.load(requestsAndSpecs[0][1])
self.assertEqual(workloadHelper.getOwner()["Requestor"], "me")
self.assertEqual(self.jsonSender.get("assignment?request=%s" % requestName)[0], ["White Sox"])
self.assertEqual(self.jsonSender.get("request/%s" % requestName)[0]["teams"], ["White Sox"])
agentUrl = "http://cmssrv96.fnal.gov/workqueue"
self.jsonSender.put("workQueue/%s?url=%s" % (requestName, urllib.quote(agentUrl)))
self.assertEqual(self.jsonSender.get("workQueue/%s" % requestName)[0][0], agentUrl)
request = self.jsonSender.get("request/%s" % requestName)[0]
self.assertEqual(request["RequestStatus"], "acquired")
self.jsonSender.post("request/%s?events_written=10&files_merged=1" % requestName)
self.jsonSender.post("request/%s?events_written=20&files_merged=2&percent_success=99.9" % requestName)
request = self.jsonSender.get("request/%s" % requestName)[0]
self.assertEqual(len(request["RequestUpdates"]), 2)
self.assertEqual(request["RequestUpdates"][0]["files_merged"], 1)
self.assertEqual(request["RequestUpdates"][1]["events_written"], 20)
self.assertEqual(request["RequestUpdates"][1]["percent_success"], 99.9)
message = "The sheriff is near"
jsonMessage = json.dumps(message)
self.jsonSender.put("message/%s" % requestName, message)
messages = self.jsonSender.get("message/%s" % requestName)
# self.assertEqual(messages[0][0][0], message)
for status in ["running", "completed"]:
self.jsonSender.put("request/%s?status=%s" % (requestName, status))
# campaign
self.jsonSender.put("campaign/%s" % "TestCampaign")
campaigns = self.jsonSender.get("campaign")[0]
self.assertTrue("TestCampaign" in campaigns.keys())
self.jsonSender.put("campaign/%s/%s" % ("TestCampaign", requestName))
requestsInCampaign = self.jsonSender.get("campaign/%s" % "TestCampaign")[0]
self.assertTrue(requestName in requestsInCampaign.keys())
req = self.jsonSender.get("request/%s" % requestName)[0]
self.assertEqual(req["Campaign"], "TestCampaign")
self.jsonSender.delete("request/%s" % requestName)
return
@attr("integration")
def testC_404Errors(self):
"""
_404Errors_
Do some things that generate 404 errors. This
should be limited to requests for objects that do not
exist.
"""
badName = "ThereIsNoWayThisNameShouldExist"
# First, try to find a non-existant request
# This should throw a 404 error.
# The request name should not be in it
self.checkForError(cls="request", badName=badName, exitCode=404, message="Given requestName not found")
# Now look for non-existant user
self.checkForError(cls="user", badName=badName, exitCode=404, message="Cannot find user")
# Now try non-existant group
self.checkForError(cls="group", badName=badName, exitCode=404, message="Cannot find group/group priority")
# Now try non-existant campaign
self.checkForError(cls="campaign", badName=badName, exitCode=404, message="Cannot find campaign")
# Now try invalid message
# This raises a requestName error becuase it searches for the request
self.checkForError(
cls="message", badName=badName, exitCode=404, message="Given requestName not found", testEmpty=False
)
# Check for assignments (no teams or requests)
# This raises a team error because it tries to load teams out first
self.checkForError(cls="assignment", badName=badName, exitCode=404, message="Cannot find team")
return
@attr("integration")
def testD_400Errors(self):
"""
_400Errors_
These are failures created by invalid input, such as sending
args to a request when it doesn't accept any. They should
generatore 400 Errors
"""
badName = "ThereIsNoWayThisNameShouldExist"
# Attempt to send arguments to a function that doesn't accept them.
self.checkForError(
cls="team", badName=badName, exitCode=400, message="Invalid input: Arguments added where none allowed"
)
# Recheck for versions
self.checkForError(
cls="version", badName=badName, exitCode=400, message="Invalid input: Arguments added where none allowed"
)
# Break the validation
self.checkForError(cls="user", badName="!", exitCode=400, message="Invalid input: Input data failed validation")
return
def checkForError(self, cls, badName, exitCode, message, testEmpty=True):
"""
_checkForError_
Generic function for checking for errors in JSON commands
Does a basic check on type cls searching for name badName which hopefull
does not exist.
Checks to make sure that it exits with code exitCode, and that
the error contains the string message.
Also checks to make sure that name badName is NOT in the output
testEmpty for those that don't handle calls to the main (i.e., who require
an argument)
"""
raises = False
# First assert that the test to be tested is empty
if testEmpty:
result = self.jsonSender.get(cls)
self.assertTrue(type(result[0]) in [type([]), type({})])
# Next, test
try:
result = self.jsonSender.get("%s/%s" % (cls, badName))
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, exitCode)
self.assertTrue(message in ex.result)
self.assertFalse(badName in ex.result)
self.assertTrue(raises)
return
def __call__(self, filesetToProcess):
"""
The algorithm itself
"""
global LOCK
# Get configuration
initObj = WMInit()
initObj.setLogging()
initObj.setDatabaseConnection(os.getenv("DATABASE"), os.getenv("DIALECT"), os.getenv("DBSOCK"))
myThread = threading.currentThread()
daofactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi)
locationNew = daofactory(classname="Locations.New")
getFileLoc = daofactory(classname="Files.GetLocation")
fileInFileset = daofactory(classname="Files.InFileset")
logging.debug("the T0Feeder is processing %s" % filesetToProcess.name)
logging.debug("the fileset name %s" % (filesetToProcess.name).split(":")[0])
# Get the start Run if asked
startRun = (filesetToProcess.name).split(":")[3]
fileType = (filesetToProcess.name).split(":")[2]
LASTIME = filesetToProcess.lastUpdate
# url builder
primaryDataset = ((filesetToProcess.name).split(":")[0]).split("/")[1]
processedDataset = ((filesetToProcess.name).split(":")[0]).split("/")[2]
dataTier = ((filesetToProcess.name).split(":")[0]).split("/")[3]
url = "/tier0/listfilesoverinterval/%s/%s/%s/%s/%s" % (
fileType,
LASTIME,
primaryDataset,
processedDataset,
dataTier,
)
tries = 1
while True:
try:
myRequester = JSONRequests(url="vocms52.cern.ch:8889")
requestResult = myRequester.get(url + "/" + "?return_type=text/json%2Bdas")
newFilesList = requestResult[0]["results"]
except:
logging.debug("T0Reader call error...")
if tries == self.maxRetries:
return
else:
tries += 1
continue
logging.debug("T0 queries done ...")
now = time.time()
LASTIME = int(newFilesList["end_time"]) + 1
break
# process all files
if len(newFilesList["files"]):
try:
locationNew.execute(siteName="caf.cern.ch", seName="caf.cern.ch")
except Exception, e:
logging.debug("Error when adding new location...")
logging.debug(e)
logging.debug(format_exc())
for files in newFilesList["files"]:
# Assume parents aren't asked
newfile = File(str(files["lfn"]), size=files["file_size"], events=files["events"])
try:
LOCK.acquire()
if newfile.exists() == False:
newfile.create()
for run in files["runs"]:
runSet = set()
runSet.add(Run(run, *files["runs"][run]))
newfile.addRunSet(runSet)
else:
newfile.loadData()
fileLoc = getFileLoc.execute(file=files["lfn"])
if "caf.cern.ch" not in fileLoc:
newfile.setLocation("caf.cern.ch")
# else:
# logging.debug("File already associated to %s" %fileLoc)
LOCK.release()
if len(newfile["runs"]):
val = 0
for run in newfile["runs"]:
if run.run < int(startRun):
val = 1
break
if not val:
listFile = fileInFileset.execute(filesetToProcess.id)
if {"fileid": newfile["id"]} not in listFile:
filesetToProcess.addFile(newfile)
filesetToProcess.setLastUpdate(int(newFilesList["end_time"]) + 1)
filesetToProcess.commit()
logging.debug("new file created/loaded added by T0AST...")
except Exception, e:
logging.debug("Error when adding new files in T0AST...")
logging.debug(e)
logging.debug(format_exc())
LOCK.release()
class ReqMgrTest(RESTBaseUnitTest):
"""
_ReqMgrTest_
Basic test for the ReqMgr services.
Setup is done off-screen in RESTBaseUnitTest - this makes
things confusing
"""
def setUp(self):
"""
setUP global values
Database setUp is done in base class
"""
self.couchDBName = "reqmgr_t_0"
RESTBaseUnitTest.setUp(self)
self.testInit.setupCouch("%s" % self.couchDBName,
"GroupUser", "ConfigCache")
self.testInit.setupCouch("%s_wmstats" % self.couchDBName,
"WMStats")
reqMgrHost = self.config.getServerUrl()
self.jsonSender = JSONRequests(reqMgrHost)
def initialize(self):
self.config = RequestManagerConfig(
'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel')
self.config.setFormatter('WMCore.WebTools.RESTFormatter')
self.config.setupRequestConfig()
self.config.setupCouchDatabase(dbName = self.couchDBName)
self.config.setPort(12888)
self.schemaModules = ["WMCore.RequestManager.RequestDB"]
def tearDown(self):
"""
tearDown
Tear down everything
"""
RESTBaseUnitTest.tearDown(self)
self.testInit.tearDownCouch()
def createConfig(self, bad = False):
"""
_createConfig_
Create a config of some sort that we can load out of ConfigCache
"""
PSetTweak = {'process': {'outputModules_': ['ThisIsAName'],
'ThisIsAName': {'dataset': {'dataTier': 'RECO',
'filterName': 'Filter'}}}}
BadTweak = {'process': {'outputModules_': ['ThisIsAName1', 'ThisIsAName2'],
'ThisIsAName1': {'dataset': {'dataTier': 'RECO',
'filterName': 'Filter'}},
'ThisIsAName2': {'dataset': {'dataTier': 'RECO',
'filterName': 'Filter'}}}}
configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = self.couchDBName)
configCache.createUserGroup(groupname = "testGroup", username = '******')
if bad:
configCache.setPSetTweaks(PSetTweak = BadTweak)
else:
configCache.setPSetTweaks(PSetTweak = PSetTweak)
configCache.save()
return configCache.getCouchID()
@attr("integration")
def testA_testBasicSetUp(self):
"""
_testBasicSetUp_
Moving the tests that were in the setUp category out of it,
mostly because I want to make sure that they don't fail inside
the setUp statement.
"""
if 'me' in self.jsonSender.get('user')[0]:
self.jsonSender.delete('user/me')
self.assertFalse('me' in self.jsonSender.get('user')[0])
self.assertEqual(self.jsonSender.put('user/[email protected]')[1], 200)
self.assertTrue('me' in self.jsonSender.get('user')[0])
if 'PeopleLikeMe' in self.jsonSender.get('group')[0]:
self.jsonSender.delete('group/PeopleLikeMe')
self.assertFalse('PeopleLikeMe' in self.jsonSender.get('group')[0])
self.assertEqual(self.jsonSender.put('group/PeopleLikeMe')[1], 200)
self.assertTrue( 'PeopleLikeMe' in self.jsonSender.get('group')[0])
self.jsonSender.put('group/PeopleLikeMe/me')
users = self.jsonSender.get('group/PeopleLikeMe')[0]['users']
self.assertTrue('me' in users)
groups = self.jsonSender.get('user/me')[0]['groups']
self.assertTrue('PeopleLikeMe' in groups)
groups2 = self.jsonSender.get('group?user=me')[0]
self.assertTrue('PeopleLikeMe' in groups2)
if 'White Sox' in self.jsonSender.get('team')[0]:
self.jsonSender.delete(urllib.quote('team/White Sox'))
self.assertFalse('White Sox' in self.jsonSender.get('team')[0])
self.assertEqual(self.jsonSender.put(urllib.quote('team/White Sox'))[1], 200)
self.assertTrue('White Sox' in self.jsonSender.get('team')[0])
# some foreign key stuff to dealwith
schema = utils.getSchema()
version = "version/" + schema["CMSSWVersion"]
self.assertTrue(self.jsonSender.put(version)[1] == 200)
self.assertTrue(schema["CMSSWVersion"] in self.jsonSender.get('version')[0])
@attr("integration")
def testB_ReReco(self):
"""
_ReReco_
Try a basic ReReco workflow
"""
schema = utils.getAndSetupSchema(self)
schema['RequestNumEvents'] = 100
schema['RequestEventSize'] = 101
self.doRequest(schema)
def doRequest(self, schema):
"""
_doRequest_
Run all tests on a basic ReReco workflow
"""
requestName = schema['RequestName']
self.assertRaises(HTTPException, self.jsonSender.delete, 'request/%s' % requestName)
result = self.jsonSender.put('request/%s' % (requestName), schema)
self.assertEqual(result[1], 200)
requestName = result[0]['RequestName']
self.assertEqual(self.jsonSender.get('request/%s' % requestName)[0]['RequestName'], requestName)
self.jsonSender.put('request/%s?status=assignment-approved' % requestName)
me = self.jsonSender.get('user/me')[0]
self.assertTrue(requestName in me['requests'])
self.assertEqual(self.jsonSender.put('request/%s?priority=5' % requestName)[1], 200)
self.assertEqual(self.jsonSender.post('user/me?priority=6')[1], 200)
self.assertEqual(self.jsonSender.post('group/PeopleLikeMe?priority=7')[1], 200)
# default priority of group and user of 1
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(request['ReqMgrRequestBasePriority'], 5)
self.assertEqual(request['ReqMgrRequestorBasePriority'], 6)
self.assertEqual(request['ReqMgrGroupBasePriority'], 7)
self.assertEqual(request['RequestPriority'], 5+6+7)
# Check LFN Bases
self.assertEqual(request['UnmergedLFNBase'], '/store/unmerged')
self.assertEqual(request['MergedLFNBase'], '/store/data')
# Check random other
self.assertEqual(request['CustodialSite'], 'US_T1_FNAL')
# Check Num events
self.assertEqual(request['RequestNumEvents'], 100)
self.assertEqual(request['RequestEventSize'], 101)
# only certain transitions allowed
#self.assertEqual(self.jsonSender.put('request/%s?status=running' % requestName)[1], 400)
self.assertRaises(HTTPException, self.jsonSender.put,'request/%s?status=running' % requestName)
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(request['RequestStatus'], 'assignment-approved')
self.assertTrue(self.jsonSender.put(urllib.quote('assignment/White Sox/%s' % requestName))[1] == 200)
requestsAndSpecs = self.jsonSender.get(urllib.quote('assignment/White Sox'))[0]
self.assertTrue(requestName in requestsAndSpecs[0])
workloadHelper = WMWorkloadHelper()
workloadHelper.load(requestsAndSpecs[0][1])
self.assertEqual(workloadHelper.getOwner()['Requestor'], "me")
self.assertEqual(self.jsonSender.get('assignment?request=%s'% requestName)[0], ['White Sox'])
self.assertEqual(self.jsonSender.get('request/%s' % requestName)[0]['teams'], ['White Sox'])
agentUrl = 'http://cmssrv96.fnal.gov/workqueue'
self.jsonSender.put('workQueue/%s?url=%s'% (requestName, urllib.quote(agentUrl)) )
self.assertEqual(self.jsonSender.get('workQueue/%s' % requestName)[0][0], agentUrl)
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(request['RequestStatus'], 'acquired')
self.jsonSender.post('request/%s?events_written=10&files_merged=1' % requestName)
self.jsonSender.post('request/%s?events_written=20&files_merged=2&percent_success=99.9' % requestName)
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(len(request['RequestUpdates']), 2)
self.assertEqual(request['RequestUpdates'][0]['files_merged'], 1)
self.assertEqual(request['RequestUpdates'][1]['events_written'], 20)
self.assertEqual(request['RequestUpdates'][1]['percent_success'], 99.9)
message = "The sheriff is near"
jsonMessage = json.dumps(message)
self.jsonSender.put('message/%s' % requestName, message)
messages = self.jsonSender.get('message/%s' % requestName)
#self.assertEqual(messages[0][0][0], message)
for status in ['running', 'completed']:
self.jsonSender.put('request/%s?status=%s' % (requestName, status))
# campaign
self.jsonSender.put('campaign/%s' % 'TestCampaign')
campaigns = self.jsonSender.get('campaign')[0]
self.assertTrue('TestCampaign' in campaigns.keys())
self.jsonSender.put('campaign/%s/%s' % ('TestCampaign', requestName))
requestsInCampaign = self.jsonSender.get('campaign/%s' % 'TestCampaign')[0]
self.assertTrue(requestName in requestsInCampaign.keys())
req = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(req['Campaign'], 'TestCampaign')
self.jsonSender.delete('request/%s' % requestName)
@attr("integration")
def testC_404Errors(self):
"""
_404Errors_
Do some things that generate 404 errors. This
should be limited to requests for objects that do not
exist.
"""
badName = 'ThereIsNoWayThisNameShouldExist'
# First, try to find a non-existant request
# This should throw a 404 error.
# The request name should not be in it
self.checkForError(cls = 'request', badName = badName, exitCode = 404,
message = 'Given requestName not found')
# Now look for non-existant user
self.checkForError(cls = 'user', badName = badName, exitCode = 404,
message = 'Cannot find user')
# Now try non-existant group
self.checkForError(cls = 'group', badName = badName, exitCode = 404,
message = "Cannot find group/group priority")
# Now try non-existant campaign
self.checkForError(cls = 'campaign', badName = badName, exitCode = 404,
message = "Cannot find campaign")
# Now try invalid message
# This raises a requestName error becuase it searches for the request
self.checkForError(cls = 'message', badName = badName, exitCode = 404,
message = "Given requestName not found", testEmpty = False)
# Check for assignments (no teams or requests)
# This raises a team error because it tries to load teams out first
self.checkForError(cls = 'assignment', badName = badName, exitCode = 404,
message = 'Cannot find team')
@attr("integration")
def testD_400Errors(self):
"""
_400Errors_
These are failures created by invalid input, such as sending
args to a request when it doesn't accept any. They should
generatore 400 Errors
"""
badName = 'ThereIsNoWayThisNameShouldExist'
# Attempt to send arguments to a function that doesn't accept them.
self.checkForError(cls = 'team', badName = badName, exitCode = 400,
message = "Invalid input: Arguments added where none allowed")
# Recheck for versions
self.checkForError(cls = 'version', badName = badName, exitCode = 400,
message = "Invalid input: Arguments added where none allowed")
# Break the validation
self.checkForError(cls = 'user', badName = '!', exitCode = 400,
message = 'Invalid input: Input data failed validation')
def checkForError(self, cls, badName, exitCode, message, testEmpty = True):
"""
_checkForError_
Generic function for checking for errors in JSON commands
Does a basic check on type cls searching for name badName which hopefull
does not exist.
Checks to make sure that it exits with code exitCode, and that
the error contains the string message.
Also checks to make sure that name badName is NOT in the output
testEmpty for those that don't handle calls to the main (i.e., who require
an argument)
"""
raises = False
# First assert that the test to be tested is empty
if testEmpty:
result = self.jsonSender.get(cls)
self.assertTrue(type(result[0]) in [type([]), type({})])
# Next, test
try:
result = self.jsonSender.get('%s/%s' % (cls, badName))
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, exitCode)
self.assertTrue(message in ex.result)
self.assertFalse(badName in ex.result)
self.assertTrue(raises)
class DBSBlock:
"""
DBSBlock
Class for holding all the necessary equipment for a DBSBlock
"""
def __init__(self, name, location, das, workflow):
"""
Just the necessary objects
Expects:
name: The blockname in full
location: The SE-name of the site the block is at
"""
self.data = {
'dataset_conf_list': [], # List of dataset configurations
'file_conf_list': [], # List of files, the configuration for each
'files': [], # List of file objects
'block': {}, # Dict of block info
'processing_era': {}, # Dict of processing era info
'acquisition_era': {}, # Dict of acquisition era information
'primds': {}, # Dict of primary dataset info
'dataset': {}, # Dict of processed dataset info
'file_parent_list': [], # List of file parents
'close_settings': {}
} # Dict of info about block close settings
self.files = []
self.encoder = JSONRequests()
self.status = 'Open'
self.inBuff = False
self.startTime = time.time()
self.name = name
self.location = location
self.das = das
self.workflow = workflow
self.data['block']['block_name'] = name
self.data['block']['origin_site_name'] = location
self.data['block']['open_for_writing'] = 1
self.data['block']['create_by'] = "WMAgent"
self.data['block']['creation_date'] = int(time.time())
self.data['block']['block_size'] = 0
self.data['block']['file_count'] = 0
self.data['block']['block_events'] = 0
self.data['close_settings'] = {}
self.data['close_settings']['block_close_max_wait_time'] = None
self.data['close_settings']['block_close_max_events'] = None
self.data['close_settings']['block_close_max_size'] = None
self.data['close_settings']['block_close_max_files'] = None
return
def encode(self):
"""
_encode_
Turn this into a JSON object for transmission
to DBS
"""
return self.encoder.encode(data=self.data)
def addFile(self, dbsFile, datasetType, primaryDatasetType):
"""
_addFile_
Add a DBSBufferFile object to our block
"""
if dbsFile['id'] in [x['id'] for x in self.files]:
msg = "Duplicate file inserted into DBSBlock: %i\n" % (
dbsFile['id'])
msg += "Ignoring this file for now!\n"
logging.error(msg)
logging.debug("Block length: %i" % len(self.files))
l = [x['id'] for x in self.files]
l.sort()
logging.debug("First file: %s Last file: %s" % (l[0], l[-1]))
return
for setting in self.data['close_settings']:
if self.data['close_settings'][setting] is None:
self.data['close_settings'][setting] = dbsFile[setting]
self.files.append(dbsFile)
self.data['block']['block_size'] += int(dbsFile['size'])
self.data['block']['file_count'] += 1
self.data['block']['block_events'] += int(dbsFile['events'])
# Assemble information for the file itself
fileDict = {}
fileDict['file_type'] = 'EDM'
fileDict['logical_file_name'] = dbsFile['lfn']
fileDict['file_size'] = dbsFile['size']
fileDict['event_count'] = dbsFile['events']
fileDict['adler32'] = "NOTSET"
fileDict['md5'] = "NOTSET"
fileDict['last_modified_by'] = "WMAgent"
fileDict['last_modification_date'] = int(time.time())
fileDict['auto_cross_section'] = 0.0
# Do the checksums
for cktype in dbsFile['checksums'].keys():
cksum = dbsFile['checksums'][cktype]
if cktype.lower() == 'cksum':
fileDict['check_sum'] = cksum
elif cktype.lower() == 'adler32':
fileDict['adler32'] = cksum
elif cktype.lower() == 'md5':
fileDict['md5'] = cksum
# Do the runs
lumiList = []
for run in dbsFile.getRuns():
for lumi in run.lumis:
lumiList.append({'lumi_section_num': lumi, 'run_num': run.run})
fileDict['file_lumi_list'] = lumiList
# Append to the files list
self.data['files'].append(fileDict)
# now add file to data
parentLFNs = dbsFile.getParentLFNs()
for lfn in parentLFNs:
self.addFileParent(child=dbsFile['lfn'], parent=lfn)
# Do the algo
algo = self.addConfiguration(release=dbsFile['appVer'],
psetHash=dbsFile['psetHash'],
appName=dbsFile['appName'],
outputLabel=dbsFile['appFam'],
globalTag=dbsFile['globalTag'])
# Now add the file with the algo
# Try to avoid messing with pointers here
fileAlgo = {}
fileAlgo.update(algo)
fileAlgo['lfn'] = dbsFile['lfn']
self.data['file_conf_list'].append(fileAlgo)
if dbsFile.get('acquisition_era', False):
self.setAcquisitionEra(dbsFile['acquisition_era'])
elif dbsFile.get('acquisitionEra', False):
self.setAcquisitionEra(dbsFile['acquisitionEra'])
if dbsFile.get('processingVer', False):
self.setProcessingVer(dbsFile['processingVer'])
elif dbsFile.get('processing_ver', False):
self.setProcessingVer(dbsFile['processing_ver'])
# Take care of the dataset
self.setDataset(datasetName=dbsFile['datasetPath'],
primaryType=primaryDatasetType,
datasetType=datasetType,
physicsGroup=dbsFile.get('physicsGroup', None),
prep_id=dbsFile.get('prep_id', None))
return
def addFileParent(self, child, parent):
"""
_addFileParent_
Add file parents to the data block
"""
info = {'parent_logical_file_name': parent, 'logical_file_name': child}
self.data['file_parent_list'].append(info)
return
def addBlockParent(self, parent):
"""
_addBlockParent_
Add the parents of the block
"""
self.data['block_parent_list'].append({'block_name': parent})
return
def addDatasetParent(self, parent):
"""
_addDatasetParent_
Add the parent datasets to the data block
"""
self.data['ds_parent_list'].append({'parent_dataset': parent})
return
def setProcessingVer(self, procVer):
"""
_setProcessingVer_
Set the block's processing version.
"""
if procVer.count("-") == 1:
(junk, self.data["processing_era"]["processing_version"]
) = procVer.split("-v")
else:
self.data["processing_era"]["processing_version"] = procVer
self.data["processing_era"]["create_by"] = "WMAgent"
self.data["processing_era"]["description"] = ""
return
def setAcquisitionEra(self, era, date=123456789):
"""
_setAcquisitionEra_
Set the acquisition era for the block
"""
self.data['acquisition_era']['acquisition_era_name'] = era
self.data['acquisition_era']['start_date'] = date
return
def setPhysicsGroup(self, group):
"""
_setPhysicsGroup_
Sets the name of the physics group to which the dataset is attached
"""
self.data['dataset']['physics_group_name'] = group
return
def hasDataset(self):
"""
_hasDataset_
Check and see if the dataset has been properly set
"""
return self.data['dataset'].get('dataset', False)
def setDataset(self,
datasetName,
primaryType,
datasetType,
physicsGroup=None,
prep_id=None,
overwrite=False,
valid=1):
"""
_setDataset_
Set all the information concerning a single dataset, including
the primary, processed and tier info
"""
if self.hasDataset() and not overwrite:
# Do nothing, we already have a dataset
return
Lexicon.primaryDatasetType(primaryType)
if not datasetType in [
'VALID', 'PRODUCTION', 'INVALID', 'DEPRECATED', 'DELETED'
]:
msg = "Invalid processedDatasetType %s\n" % datasetType
logging.error(msg)
raise DBSBlockException(msg)
try:
if datasetName[0] == '/':
junk, primary, processed, tier = datasetName.split('/')
else:
primary, processed, tier = datasetName.split('/')
except Exception, ex:
msg = "Invalid dataset name %s" % datasetName
logging.error(msg)
raise DBSBlockException(msg)
# Do the primary dataset
self.data['primds']['primary_ds_name'] = primary
self.data['primds']['primary_ds_type'] = primaryType
self.data['primds']['create_by'] = "WMAgent"
self.data['primds']['creation_date'] = int(time.time())
# Do the processed
self.data['dataset']['physics_group_name'] = physicsGroup
self.data['dataset']['processed_ds_name'] = processed
self.data['dataset']['data_tier_name'] = tier
self.data['dataset']['dataset_access_type'] = datasetType
self.data['dataset']['dataset'] = datasetName
self.data['dataset']['prep_id'] = prep_id
# Add misc meta data.
self.data['dataset']['create_by'] = "WMAgent"
self.data['dataset']['last_modified_by'] = "WMAgent"
self.data['dataset']['creation_date'] = int(time.time())
self.data['dataset']['last_modification_date'] = int(time.time())
return
class ReqMgrPriorityTest(RESTBaseUnitTest):
"""
_ReqMgrPriorityTest_
Basic test for setting the priority in ReqMgr Services
"""
def setUp(self):
"""
setUP global values
Database setUp is done in base class
"""
self.couchDBName = "reqmgr_t_0"
RESTBaseUnitTest.setUp(self)
self.testInit.setupCouch("%s" % self.couchDBName, "GroupUser",
"ConfigCache")
self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats")
reqMgrHost = self.config.getServerUrl()
self.jsonSender = JSONRequests(reqMgrHost)
def initialize(self):
self.config = RequestManagerConfig(
'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel')
self.config.setFormatter('WMCore.WebTools.RESTFormatter')
self.config.setupRequestConfig()
self.config.setupCouchDatabase(dbName=self.couchDBName)
self.config.setPort(8888)
self.schemaModules = ["WMCore.RequestManager.RequestDB"]
def tearDown(self):
"""
_tearDown_
Basic tear down of database
"""
RESTBaseUnitTest.tearDown(self)
self.testInit.tearDownCouch()
def loadWorkload(self, requestName):
"""
_loadWorkload_
Load the workload from couch after we've saved it there.
"""
workload = WMWorkloadHelper()
url = '%s/%s/%s/spec' % (os.environ['COUCHURL'], self.couchDBName,
requestName)
workload.load(url)
return workload
def changeStatusAndCheck(self, requestName, statusName):
"""
_changeStatusAndCheck_
Change the status of a request and make sure that
the request actually did it.
"""
self.jsonSender.put('request/%s?status=%s' % (requestName, statusName))
result = self.jsonSender.get('request/%s' % requestName)
self.assertEqual(result[0]['RequestStatus'], statusName)
def testA_RequestPriority(self):
"""
_priorityChanges_
Do some fairly standard priority changes to the Request and
see how things react
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName=userName,
groupName=groupName,
teamName=teamName)
result = self.jsonSender.put('request/testRequest', schema)
self.assertEqual(result[1], 200)
requestName = result[0]['RequestName']
workload = self.loadWorkload(requestName=requestName)
self.assertEqual(workload.priority(), 0)
# Reset user, group priorities to 0
self.jsonSender.post('user/%s?priority=0' % userName)
self.jsonSender.post('group/%s?priority=0' % groupName)
# Set priority == 5
priority = 5
self.jsonSender.put('request/%s?priority=%s' % (requestName, priority))
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(request['ReqMgrRequestBasePriority'], priority)
workload = self.loadWorkload(requestName=requestName)
self.assertEqual(workload.priority(), priority)
# Set priority == 100
priority = 100
self.jsonSender.put('request/%s?priority=%s' % (requestName, priority))
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(request['ReqMgrRequestBasePriority'], priority)
workload = self.loadWorkload(requestName=requestName)
self.assertEqual(workload.priority(), priority)
# Set priority == -1
priority = -1
self.jsonSender.put('request/%s?priority=%s' % (requestName, priority))
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(request['ReqMgrRequestBasePriority'], priority)
workload = self.loadWorkload(requestName=requestName)
self.assertEqual(workload.priority(), priority)
# Let's move the request around a bit
self.changeStatusAndCheck(requestName=requestName,
statusName='testing-approved')
self.changeStatusAndCheck(requestName=requestName,
statusName='testing')
self.changeStatusAndCheck(requestName=requestName, statusName='tested')
self.changeStatusAndCheck(requestName=requestName,
statusName='assignment-approved')
self.jsonSender.put(
urllib.quote('assignment/%s/%s' % (teamName, requestName)))
# Set priority == 99
priority = 99
self.jsonSender.put('request/%s?priority=%s' % (requestName, priority))
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(request['ReqMgrRequestBasePriority'], priority)
workload = self.loadWorkload(requestName=requestName)
self.assertEqual(workload.priority(), priority)
def testB_InvalidPriority(self):
"""
_InvalidPriority_
Put in a bunch of invalid values for priorities and
see what the code makes of them.
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName=userName,
groupName=groupName,
teamName=teamName)
result = self.jsonSender.put('request/testRequest', schema)
self.assertEqual(result[1], 200)
requestName = result[0]['RequestName']
raises = False
try:
priority = sys.maxint + 1
self.jsonSender.put('request/%s?priority=%s' %
(requestName, priority))
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue(
"Priority must have abs() less then MAXINT!" in ex.result)
self.assertTrue(raises)
raises = False
try:
priority = -1 - sys.maxint
self.jsonSender.put('request/%s?priority=%s' %
(requestName, priority))
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
print ex.result
self.assertTrue(
"Priority must have abs() less then MAXINT!" in ex.result)
if not os.path.exists(configPath):
# We can do nothing -
logging.error("Something in the way of the config path")
sys.exit(1)
f = open(configPath, 'r')
config = cPickle.load(f)
f.close()
# Setup DB
wmInit = WMInit()
setupDB(config, wmInit)
# Create JSON handler
jsonHandler = JSONRequests()
wmFactory = WMFactory(name = "slaveFactory", namespace = namespace)
slaveClass = wmFactory.loadObject(classname = slaveClassName, args = config)
logging.info("Have slave class")
while(True):
encodedInput = receiver.recv()
try:
input = jsonHandler.decode(encodedInput)
except Exception as ex:
logging.error("Error decoding: %s" % str(ex))
break
class ProcessPool:
def __init__(self, slaveClassName, totalSlaves, componentDir,
config, namespace = 'WMComponent', inPort = '5555',
outPort = '5558'):
"""
__init__
Constructor for the process pool. The slave class name must be based
inside the WMComponent namespace. For examples, the JobAccountant would
pass in 'JobAccountant.AccountantWorker' to run the AccountantWorker
class. All log files will be stored in the component directory that is
passed in. Each slave will have its own log file.
Note that the config is only used to determine database connection
parameters. It is not passed to the slave class. The slaveInit
parameter will be serialized and passed to the slave class's
constructor.
"""
self.enqueueIndex = 0
self.dequeueIndex = 0
self.runningWork = 0
#Use the Services.Requests JSONizer, which handles __to_json__ calls
self.jsonHandler = JSONRequests()
# heartbeat should be registered at this point
if getattr(config.Agent, "useHeartbeat", True):
self.heartbeatAPI = HeartbeatAPI(getattr(config.Agent, "componentName", "ProcPoolSlave"))
self.slaveClassName = slaveClassName
self.componentDir = componentDir
self.config = config
# Grab the python version from the current version
# Assume naming convention pythonA.B, i.e., python2.4 for v2.4.X
majorVersion = sys.version_info[0]
minorVersion = sys.version_info[1]
if majorVersion and minorVersion:
self.versionString = "python%i.%i" % (majorVersion, minorVersion)
else:
self.versionString = "python2.6"
self.workers = []
self.nSlaves = totalSlaves
self.namespace = namespace
self.inPort = inPort
self.outPort = outPort
# Pickle the config
self.configPath = os.path.join(componentDir, '%s_config.pkl' % slaveClassName)
if os.path.exists(self.configPath):
# Then we note it and overwrite it
msg = "Something's in the way of the ProcessPool config: %s" % self.configPath
logging.error(msg)
f = open(self.configPath, 'w')
cPickle.dump(config, f)
f.close()
# Set up ZMQ
try:
context = zmq.Context()
self.sender = context.socket(zmq.PUSH)
self.sender.bind("tcp://*:%s" % inPort)
self.sink = context.socket(zmq.PULL)
self.sink.bind("tcp://*:%s" % outPort)
except zmq.ZMQError:
# Try this again in a moment to see
# if it's just being held by something pre-existing
import time
time.sleep(1)
logging.error("Blocked socket on startup: Attempting sleep to give it time to clear.")
try:
context = zmq.Context()
self.sender = context.socket(zmq.PUSH)
self.sender.bind("tcp://*:%s" % inPort)
self.sink = context.socket(zmq.PULL)
self.sink.bind("tcp://*:%s" % outPort)
except Exception as ex:
msg = "Error attempting to open TCP sockets\n"
msg += str(ex)
logging.error(msg)
import traceback
print traceback.format_exc()
raise ProcessPoolException(msg)
# Now actually create the slaves
self.createSlaves()
return
def createSlaves(self):
"""
_createSlaves_
Create the slaves by using the values from __init__()
Moving it into a separate function allows us to restart
all of them.
"""
totalSlaves = self.nSlaves
slaveClassName = self.slaveClassName
config = self.config
namespace = self.namespace
inPort = self.inPort
outPort = self.outPort
slaveArgs = [self.versionString, __file__, self.slaveClassName, inPort,
outPort, self.configPath, self.componentDir, self.namespace]
count = 0
while totalSlaves > 0:
#For each worker you want create a slave process
#That process calls this code (WMCore.ProcessPool) and opens
#A process pool that loads the designated class
slaveProcess = subprocess.Popen(slaveArgs, stdin = subprocess.PIPE,
stdout = subprocess.PIPE)
self.workers.append(slaveProcess)
totalSlaves -= 1
count += 1
return
def _subProcessName(self, slaveClassName, sequence):
""" subProcessName for heartbeat
could change to use process ID as a suffix
"""
return "%s_%s" % (slaveClassName, sequence + 1)
def __del__(self):
"""
__del__
Kill all the workers processes by sending them an invalid JSON object.
This will cause them to shut down.
"""
self.close()
return
def close(self):
"""
_close_
Close shuts down all the active systems by:
a) Sending STOP commands for all workers
b) Closing the pipes
c) Shutting down the workers themselves
"""
for i in range(self.nSlaves):
try:
encodedWork = self.jsonHandler.encode('STOP')
self.sender.send(encodedWork)
except Exception as ex:
# Might be already failed. Nothing you can
# really do about that.
logging.error("Failure killing running process: %s" % str(ex))
pass
try:
self.sender.close()
except:
# We can't really do anything if we fail
pass
try:
self.sink.close()
except:
# We can't do anything if we fail
pass
# Now close the workers by hand
for worker in self.workers:
try:
worker.join()
except Exception as ex:
try:
worker.terminate()
except Exception as ex2:
logging.error("Failure to join or terminate process")
logging.error(str(ex))
logging.error(str(ex2))
continue
self.workers = []
return
def enqueue(self, work, list = False):
"""
__enqeue__
Assign work to the workers processes. The work parameters must be a
list where each item in the list can be serialized into JSON.
If list is True, the entire list is sent as one piece of work
"""
if len(self.workers) < 1:
# Someone's shut down the system
msg = "Attempting to send work after system failure and shutdown!\n"
logging.error(msg)
raise ProcessPoolException(msg)
if not list:
for w in work:
encodedWork = self.jsonHandler.encode(w)
self.sender.send(encodedWork)
self.runningWork += 1
else:
encodedWork = self.jsonHandler.encode(work)
self.sender.send(encodedWork)
self.runningWork += 1
return
def dequeue(self, totalItems = 1):
"""
__dequeue__
Retrieve completed work from the slave workers. This method will block
until enough work has been completed.
"""
completedWork = []
if totalItems > self.runningWork:
msg = "Asked to dequeue more work then is running!\n"
msg += "Failing"
logging.error(msg)
raise ProcessPoolException(msg)
while totalItems > 0:
try:
output = self.sink.recv()
decode = self.jsonHandler.decode(output)
if type(decode) == type({}) and decode.get('type', None) == 'ERROR':
# Then we had some kind of error
msg = decode.get('msg', 'Unknown Error in ProcessPool')
logging.error("Received Error Message from ProcessPool Slave")
logging.error(msg)
self.close()
raise ProcessPoolException(msg)
completedWork.append(decode)
self.runningWork -= 1
totalItems -= 1
except Exception as ex:
msg = "Exception while getting slave outputin ProcessPool.\n"
msg += str(ex)
logging.error(msg)
break
return completedWork
def restart(self):
"""
_restart_
Delete everything and restart all pools
"""
self.close()
self.createSlaves()
return
class RequestManagerTest(RESTBaseUnitTest):
"""
Test RequestMgr Service client
It will start RequestMgr RESTService
Server DB is whatever env is set
This checks whether DS call makes without error and return the results.
This test only test service call returns without error.
The correctness of each function is tested in test/python/RequestManager_t/RequestMgr_t.py
"""
def initialize(self):
self.couchDBName = "reqmgr_t_0"
self.config = RequestManagerConfig(
'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel')
dbUrl = os.environ.get("DATABASE", None)
self.config.setDBUrl(dbUrl)
self.config.setFormatter('WMCore.WebTools.RESTFormatter')
self.config.setupRequestConfig()
self.config.setupCouchDatabase(dbName = self.couchDBName)
self.config.setPort(8899)
self.schemaModules = ["WMCore.RequestManager.RequestDB"]
def setUp(self):
RESTBaseUnitTest.setUp(self)
self.testInit.setupCouch("%s" % self.couchDBName, "GroupUser", "ConfigCache", "ReqMgr")
self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats")
# logging stuff from TestInit is broken, setting myself
l = logging.getLogger()
l.setLevel(logging.DEBUG)
self.params = {}
self.params['endpoint'] = self.config.getServerUrl()
self.reqService = RequestManagerDS(self.params)
self.jsonSender = JSONRequests(self.config.getServerUrl())
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
schema['ConfigCacheID'] = self.createConfig()
schema['CouchDBName'] = self.couchDBName
schema['CouchWorkloadDBName'] = self.couchDBName
try:
r = self.jsonSender.put('request', schema)
try:
self.requestName = r[0]['RequestName']
except:
self.requestName = r[0].values()[0]['RequestName']
except Exception as ex:
msg = traceback.format_exc()
print("Exception during set up, reason: %s" % msg)
raise ex
def tearDown(self):
self.config.deleteWorkloadCache()
RESTBaseUnitTest.tearDown(self)
self.testInit.tearDownCouch()
def createConfig(self, bad = False):
"""
_createConfig_
Create a config of some sort that we can load out of ConfigCache
"""
PSetTweak = {'process': {'outputModules_': ['ThisIsAName'],
'ThisIsAName': {'dataset': {'dataTier': 'RECO',
'filterName': 'Filter'}}}}
BadTweak = {'process': {'outputModules_': ['ThisIsAName1', 'ThisIsAName2'],
'ThisIsAName1': {'dataset': {'dataTier': 'RECO',
'filterName': 'Filter'}},
'ThisIsAName2': {'dataset': {'dataTier': 'RECO',
'filterName': 'Filter'}}}}
configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = self.couchDBName)
configCache.createUserGroup(groupname = "testGroup", username = '******')
if bad:
configCache.setPSetTweaks(PSetTweak = BadTweak)
else:
configCache.setPSetTweaks(PSetTweak = PSetTweak)
configCache.save()
return configCache.getCouchID()
@attr("integration")
def testA_RequestManagerService(self):
requestName = self.requestName
request = self.reqService.getRequest(requestName)
# minimal test : it's return type and the some value inside
self.assertEqual(type(request), dict)
self.assertTrue(len(request) > 0)
# Test putTeam
self.reqService.putTeam("team_usa")
self.assertTrue('team_usa' in self.jsonSender.get('team')[0])
self.jsonSender.put('assignment/%s/%s' % ("team_usa", requestName))
request = self.reqService.getAssignment(teamName = "team_usa")
self.assertEqual(type(request), list)
self.assertTrue(len(request) > 0)
request = self.reqService.getAssignment(request = requestName)
self.assertEqual(type(request), list)
self.assertTrue(len(request) > 0)
self.reqService.sendMessage(requestName,"error")
self.reqService.putWorkQueue(requestName, "http://test_url")
self.reqService.reportRequestProgress(requestName,
percent_complete = 100, percent_success = 90)
self.reqService.updateRequestStatus(requestName, "running-open")
def testSpecialCharacterPasswords(self):
url = 'http://*****:*****@ssw:rd@localhost:6666'
req = JSONRequests(url)
self.assertEqual(req['host'], 'http://localhost:6666')
self.assertEqual(req.additionalHeaders['Authorization'], 'Basic dXNlcm5hbWU6cEBzc3c6cmQ=')
class ReqMgrWorkloadTest(RESTBaseUnitTest):
"""
Test that sets up and checks the validations of the various main WMSpec.StdSpecs
This is mostly a simple set of tests which can be very repetitive.
"""
def setUp(self):
"""
setUP global values
Database setUp is done in base class
"""
self.couchDBName = "reqmgr_t_0"
RESTBaseUnitTest.setUp(self)
self.testInit.setupCouch("%s" % self.couchDBName, "ConfigCache",
"ReqMgr")
self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats")
reqMgrHost = self.config.getServerUrl()
self.jsonSender = JSONRequests(reqMgrHost)
def initialize(self):
self.config = RequestManagerConfig(
'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel')
self.config.setFormatter('WMCore.WebTools.RESTFormatter')
self.config.setupRequestConfig()
self.config.setupCouchDatabase(dbName=self.couchDBName)
self.config.setPort(12888)
self.schemaModules = ["WMCore.RequestManager.RequestDB"]
def tearDown(self):
"""
_tearDown_
Basic tear down of database
"""
RESTBaseUnitTest.tearDown(self)
self.testInit.tearDownCouch()
def createConfig(self, bad=False):
"""
_createConfig_
Create a config of some sort that we can load out of ConfigCache
"""
PSetTweak = {
'process': {
'outputModules_': ['ThisIsAName'],
'ThisIsAName': {
'dataset': {
'dataTier': 'RECO',
'filterName': 'Filter'
}
}
}
}
BadTweak = {
'process': {
'outputModules_': ['ThisIsAName1', 'ThisIsAName2'],
'ThisIsAName1': {
'dataset': {
'dataTier': 'RECO',
'filterName': 'Filter'
}
},
'ThisIsAName2': {
'dataset': {
'dataTier': 'RECO',
'filterName': 'Filter'
}
}
}
}
configCache = ConfigCache(os.environ["COUCHURL"],
couchDBName=self.couchDBName)
configCache.createUserGroup(groupname="testGroup", username='******')
if bad:
configCache.setPSetTweaks(PSetTweak=BadTweak)
else:
configCache.setPSetTweaks(PSetTweak=PSetTweak)
configCache.save()
return configCache.getCouchID()
def loadWorkload(self, requestName):
"""
_loadWorkload_
Load the workload from couch after we've saved it there.
"""
workload = WMWorkloadHelper()
url = '%s/%s/%s/spec' % (os.environ['COUCHURL'], self.couchDBName,
requestName)
workload.load(url)
return workload
@attr('integration')
def testA_makeReRecoWorkload(self):
"""
_makeReRecoWorkload_
Check that you can make a basic ReReco workload
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName=userName,
groupName=groupName,
teamName=teamName)
del schema['GlobalTag']
configID = self.createConfig(bad=False)
schema['ConfigCacheID'] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
schema["CouchWorkloadDBName"] = self.couchDBName
raises = False
try:
self.jsonSender.put('request', schema)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue(
"Error in Workload Validation: Argument GlobalTag is required."
in ex.result)
self.assertTrue(raises)
schema = utils.getSchema(groupName=groupName, userName=userName)
schema['InputDataset'] = '/Nothing'
raises = False
try:
self.jsonSender.put('request', schema)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue(raises)
schema = utils.getSchema(groupName=groupName, userName=userName)
raises = False
del schema['ConfigCacheID']
try:
self.jsonSender.put('request', schema)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue(
"Create request failed, 'ConfigCacheID'" in ex.result)
self.assertTrue(raises)
configID = self.createConfig(bad=True)
schema = utils.getSchema(groupName=groupName, userName=userName)
schema["ConfigCacheID"] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
schema["CouchWorkloadDBName"] = self.couchDBName
raises = False
try:
self.jsonSender.put('request', schema)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue(
"Error in Workload Validation: "
"Duplicate dataTier/filterName combination" in ex.result)
self.assertTrue(raises)
schema = utils.getSchema(groupName=groupName, userName=userName)
configID = self.createConfig(bad=False)
schema['ConfigCacheID'] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
schema["CouchWorkloadDBName"] = self.couchDBName
try:
result = self.jsonSender.put('request', schema)
except Exception as ex:
raise
self.assertEqual(result[1], 200)
requestName = result[0]['RequestName']
result = self.jsonSender.get('request/%s' % requestName)
request = result[0]
self.assertEqual(request['CMSSWVersion'], schema['CMSSWVersion'])
self.assertEqual(request['Group'], groupName)
self.assertEqual(request['Requestor'], userName)
@attr('integration')
def testB_Analysis(self):
"""
_Analysis_
Test Analysis workflows
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName=userName,
groupName=groupName,
teamName=teamName)
schema['RequestType'] = "Analysis"
try:
raises = False
result = self.jsonSender.put('request', schema)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue(raises)
# Put the right things in the schema
# And watch it fail, because we can't open a secure connection to CERN
# in a unittest well
schema['RequestorDN'] = 'SomeDN'
#result = self.jsonSender.put('request/testRequest', schema)
#requestName = result[0]['RequestName']
#result = self.jsonSender.get('request/%s' % requestName)
#request = result[0]
#self.assertEqual(request['CMSSWVersion'], CMSSWVersion)
#self.assertEqual(request['Group'], groupName)
#self.assertEqual(request['Requestor'], userName)
@attr('integration')
def testD_ReDigi(self):
"""
_ReDigi_
Test ReDigi workflows
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName=userName,
groupName=groupName,
teamName=teamName)
schema['RequestType'] = "ReDigi"
try:
raises = False
result = self.jsonSender.put('request', schema)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue(
"Create request failed, 'StepOneConfigCacheID'" in ex.result)
self.assertTrue(raises)
schema["StepOneConfigCacheID"] = "fakeID"
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
schema["CouchWorkloadDBName"] = self.couchDBName
try:
raises = False
result = self.jsonSender.put('request', schema)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue(
"Failure to load ConfigCache while validating workload" in
ex.result)
self.assertTrue(raises)
configID = self.createConfig()
schema["StepOneConfigCacheID"] = configID
result = self.jsonSender.put('request', schema)
requestName = result[0]['RequestName']
result = self.jsonSender.get('request/%s' % requestName)
request = result[0]
self.assertEqual(request['CMSSWVersion'], schema['CMSSWVersion'])
self.assertEqual(request['Group'], groupName)
self.assertEqual(request['Requestor'], userName)
self.assertEqual(request['SizePerEvent'], 512)
self.assertEqual(request['RequestNumEvents'], 0)
self.assertEqual(request['RequestSizeFiles'], 0)
@attr('integration')
def testE_StoreResults(self):
"""
_StoreResults_
Test StoreResults workflows
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName=userName,
groupName=groupName,
teamName=teamName)
schema['RequestType'] = "StoreResults"
schema["CouchWorkloadDBName"] = self.couchDBName
try:
raises = False
result = self.jsonSender.put('request', schema)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue(
"Error in Workload Validation: Argument CmsPath is required."
in ex.result)
self.assertTrue(raises)
schema['DbsUrl'] = 'http://fake.dbs.url/dbs'
schema['CmsPath'] = '/fake/tmp/path'
schema['AcquisitionEra'] = 'era'
schema['ProcessingVersion'] = 1
result = self.jsonSender.put('request', schema)
requestName = result[0]['RequestName']
result = self.jsonSender.get('request/%s' % requestName)
request = result[0]
self.assertEqual(request['CMSSWVersion'], schema['CMSSWVersion'])
self.assertEqual(request['Group'], groupName)
self.assertEqual(request['Requestor'], userName)
self.assertEqual(request['DbsUrl'], schema['DbsUrl'])
@attr('integration')
def testF_TaskChain(self):
"""
_TaskChain_
Test the monstrous TaskChain workflow
This will be a long one
NOTE: This test is so complicated that all I do is
take code from TaskChain_t and make sure it still
produces and actual request
"""
couchServer = CouchServer(os.environ["COUCHURL"])
configDatabase = couchServer.connectDatabase(self.couchDBName)
generatorDoc = makeGeneratorConfig(configDatabase)
processorDocs = makeProcessingConfigs(configDatabase)
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getSchema(userName=userName)
schema["CouchURL"] = os.environ["COUCHURL"]
schema["CouchDBName"] = self.couchDBName
schema["CouchWorkloadDBName"] = self.couchDBName
schema["SiteWhitelist"] = ["T1_CH_CERN", "T1_US_FNAL"]
schema["TaskChain"] = 5
chains = {
"Task1": {
"TaskName": "GenSim",
"ConfigCacheID": generatorDoc,
"SplittingAlgo": "EventBased",
"EventsPerJob": 250,
"RequestNumEvents": 10000,
"PrimaryDataset": "RelValTTBar"
},
"Task2": {
"TaskName": "DigiHLT",
"InputTask": "GenSim",
"InputFromOutputModule": "writeGENSIM",
"ConfigCacheID": processorDocs['DigiHLT'],
"SplittingAlgo": "FileBased"
},
"Task3": {
"TaskName": "Reco",
"InputTask": "DigiHLT",
"InputFromOutputModule": "writeRAWDIGI",
"ConfigCacheID": processorDocs['Reco'],
"SplittingAlgo": "FileBased"
},
"Task4": {
"TaskName": "ALCAReco",
"InputTask": "Reco",
"InputFromOutputModule": "writeALCA",
"ConfigCacheID": processorDocs['ALCAReco'],
"SplittingAlgo": "FileBased"
},
"Task5": {
"TaskName": "Skims",
"InputTask": "Reco",
"InputFromOutputModule": "writeRECO",
"ConfigCacheID": processorDocs['Skims'],
"SplittingAlgo": "FileBased",
"FilesPerJob": 10
}
}
schema.update(chains)
args = utils.getAndSetupSchema(self,
userName=userName,
groupName=groupName,
teamName=teamName)
schema.update(args)
# this is necessary and after all updates to the schema are made,
# otherwise this item will get overwritten
schema['RequestType'] = "TaskChain"
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
schema["CouchWorkloadDBName"] = self.couchDBName
result = self.jsonSender.put('request', schema)
requestName = result[0]['RequestName']
result = self.jsonSender.get('request/%s' % requestName)
request = result[0]
self.assertEqual(request['CMSSWVersion'], schema['CMSSWVersion'])
self.assertEqual(request['Group'], groupName)
self.assertEqual(request['Requestor'], userName)
workload = self.loadWorkload(requestName)
self.assertEqual(workload.data.request.schema.Task1["EventsPerJob"],
250)
@attr('integration')
def testG_MonteCarloFromGEN(self):
"""
_MonteCarloFromGEN_
Test MonteCarloFromGEN workflows
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName=userName,
groupName=groupName,
teamName=teamName)
schema['RequestType'] = "MonteCarloFromGEN"
try:
raises = False
result = self.jsonSender.put('request', schema)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue(
"Error in Workload Validation: Failure to load ConfigCache while validating workload"
in ex.result)
self.assertTrue(raises)
schema["ConfigCacheID"] = "fakeID"
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
schema["CouchWorkloadDBName"] = self.couchDBName
try:
raises = False
result = self.jsonSender.put('request', schema)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue(
"Failure to load ConfigCache while validating workload" in
ex.result)
self.assertTrue(raises)
configID = self.createConfig()
schema["ConfigCacheID"] = configID
result = self.jsonSender.put('request', schema)
requestName = result[0]['RequestName']
result = self.jsonSender.get('request/%s' % requestName)
request = result[0]
self.assertEqual(request['CMSSWVersion'], schema['CMSSWVersion'])
self.assertEqual(request['Group'], groupName)
self.assertEqual(request['Requestor'], userName)
self.assertEqual(request['SizePerEvent'], 512)
self.assertEqual(request['RequestNumEvents'], 0)
self.assertEqual(request['RequestSizeFiles'], 0)
def testH_MonteCarlo(self):
"""
_MonteCarlo_
Test MonteCarlo workflows
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName=userName,
groupName=groupName,
teamName=teamName)
schema['RequestType'] = "MonteCarlo"
schema['DbsUrl'] = 'https://cmsweb.cern.ch/dbs/prod/global/DBSReader'
# Set some versions
schema['ProcessingVersion'] = '2012'
schema['AcquisitionEra'] = 'ae2012'
try:
raises = False
result = self.jsonSender.put('request', schema)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue(
"Error in Workload Validation: Failure to load ConfigCache while validating workload"
in ex.result)
self.assertTrue(raises)
schema["ConfigCacheID"] = "fakeID"
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
schema["CouchWorkloadDBName"] = self.couchDBName
schema["PrimaryDataset"] = "ReallyFake"
schema["RequestNumEvents"] = 100
try:
raises = False
result = self.jsonSender.put('request', schema)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue(raises)
configID = self.createConfig()
schema["ConfigCacheID"] = configID
schema["FilterEfficiency"] = -0.5
try:
raises = False
result = self.jsonSender.put('request', schema)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
# until exception handling is redone (New REST API)
#self.assertTrue("Negative filter efficiency for MC workflow" in ex.result)
self.assertTrue(raises)
schema["FilterEfficiency"] = 1.0
result = self.jsonSender.put('request', schema)
requestName = result[0]['RequestName']
result = self.jsonSender.get('request/%s' % requestName)
request = result[0]
self.assertEqual(request['CMSSWVersion'], schema['CMSSWVersion'])
self.assertEqual(request['Group'], groupName)
self.assertEqual(request['Requestor'], userName)
self.assertEqual(request['DbsUrl'], schema['DbsUrl'])
self.assertEqual(request['SizePerEvent'], 512)
self.assertEqual(request['RequestNumEvents'], 100)
self.assertEqual(request['RequestSizeFiles'], 0)
def testJ_Resubmission(self):
"""
_Resubmission_
Test Resubmission
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName=userName,
groupName=groupName,
teamName=teamName)
schema['RequestType'] = "ReReco"
configID = self.createConfig()
schema["ConfigCacheID"] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
schema["CouchWorkloadDBName"] = self.couchDBName
result = self.jsonSender.put('request', schema)
requestName = result[0]['RequestName']
# user, group schema already set up
schema = utils.getSchema(groupName=groupName, userName=userName)
schema['RequestType'] = "Resubmission"
schema["CouchWorkloadDBName"] = self.couchDBName
try:
raises = False
result = self.jsonSender.put('request', schema)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue(
"Error in Workload Validation: Validation failed: InitialTaskPath is mendatory"
in ex.result)
self.assertTrue(raises)
schema["InitialTaskPath"] = '/%s/DataProcessing' % requestName
schema["ACDCServer"] = os.environ.get("COUCHURL")
schema["ACDCDatabase"] = self.couchDBName
schema["CollectionName"] = "SomeOtherName"
# Here we just make sure that real result goes through
result = self.jsonSender.put('request', schema)
resubmitName = result[0]['RequestName']
result = self.jsonSender.get('request/%s' % resubmitName)
couchServer = CouchServer(self.testInit.couchUrl)
reqmgrCouch = couchServer.connectDatabase(self.couchDBName)
result = reqmgrCouch.loadView('ReqMgr', 'childresubmissionrequests',
{}, [requestName])['rows']
self.assertEqual(len(result), 1)
self.assertEqual(result[0]['key'], requestName)
self.assertEqual(result[0]['id'], resubmitName)
def __init__(self, slaveClassName, totalSlaves, componentDir,
config, namespace = 'WMComponent', inPort = '5555',
outPort = '5558'):
"""
__init__
Constructor for the process pool. The slave class name must be based
inside the WMComponent namespace. For examples, the JobAccountant would
pass in 'JobAccountant.AccountantWorker' to run the AccountantWorker
class. All log files will be stored in the component directory that is
passed in. Each slave will have its own log file.
Note that the config is only used to determine database connection
parameters. It is not passed to the slave class. The slaveInit
parameter will be serialized and passed to the slave class's
constructor.
"""
self.enqueueIndex = 0
self.dequeueIndex = 0
self.runningWork = 0
#Use the Services.Requests JSONizer, which handles __to_json__ calls
self.jsonHandler = JSONRequests()
# heartbeat should be registered at this point
if getattr(config.Agent, "useHeartbeat", True):
self.heartbeatAPI = HeartbeatAPI(getattr(config.Agent, "componentName", "ProcPoolSlave"))
self.slaveClassName = slaveClassName
self.componentDir = componentDir
self.config = config
# Grab the python version from the current version
# Assume naming convention pythonA.B, i.e., python2.4 for v2.4.X
majorVersion = sys.version_info[0]
minorVersion = sys.version_info[1]
if majorVersion and minorVersion:
self.versionString = "python%i.%i" % (majorVersion, minorVersion)
else:
self.versionString = "python2.6"
self.workers = []
self.nSlaves = totalSlaves
self.namespace = namespace
self.inPort = inPort
self.outPort = outPort
# Pickle the config
self.configPath = os.path.join(componentDir, '%s_config.pkl' % slaveClassName)
if os.path.exists(self.configPath):
# Then we note it and overwrite it
msg = "Something's in the way of the ProcessPool config: %s" % self.configPath
logging.error(msg)
f = open(self.configPath, 'w')
cPickle.dump(config, f)
f.close()
# Set up ZMQ
try:
context = zmq.Context()
self.sender = context.socket(zmq.PUSH)
self.sender.bind("tcp://*:%s" % inPort)
self.sink = context.socket(zmq.PULL)
self.sink.bind("tcp://*:%s" % outPort)
except zmq.ZMQError:
# Try this again in a moment to see
# if it's just being held by something pre-existing
import time
time.sleep(1)
logging.error("Blocked socket on startup: Attempting sleep to give it time to clear.")
try:
context = zmq.Context()
self.sender = context.socket(zmq.PUSH)
self.sender.bind("tcp://*:%s" % inPort)
self.sink = context.socket(zmq.PULL)
self.sink.bind("tcp://*:%s" % outPort)
except Exception as ex:
msg = "Error attempting to open TCP sockets\n"
msg += str(ex)
logging.error(msg)
import traceback
print traceback.format_exc()
raise ProcessPoolException(msg)
# Now actually create the slaves
self.createSlaves()
return
class ReqMgrPriorityTest(RESTBaseUnitTest):
"""
Basic test for setting the priority in ReqMgr Services
"""
def setUp(self):
"""
setUP global values
Database setUp is done in base class
"""
self.couchDBName = "reqmgr_t_0"
RESTBaseUnitTest.setUp(self)
self.testInit.setupCouch("%s" % self.couchDBName, "ConfigCache", "ReqMgr")
self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats")
self.testInit.setupCouch("%s_acdc" % self.couchDBName, "ACDC")
reqMgrHost = self.config.getServerUrl()
self.jsonSender = JSONRequests(reqMgrHost)
def initialize(self):
self.config = RequestManagerConfig(
'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel')
self.config.setFormatter('WMCore.WebTools.RESTFormatter')
self.config.setupRequestConfig()
self.config.setupCouchDatabase(dbName = self.couchDBName)
self.config.setPort(8888)
self.schemaModules = ["WMCore.RequestManager.RequestDB"]
def tearDown(self):
"""
_tearDown_
Basic tear down of database
"""
RESTBaseUnitTest.tearDown(self)
self.testInit.tearDownCouch()
def loadWorkload(self, requestName):
"""
_loadWorkload_
Load the workload from couch after we've saved it there.
"""
workload = WMWorkloadHelper()
url = '%s/%s/%s/spec' % (os.environ['COUCHURL'], self.couchDBName,
requestName)
workload.load(url)
return workload
def changeStatusAndCheck(self, requestName, statusName):
"""
_changeStatusAndCheck_
Change the status of a request and make sure that
the request actually did it.
"""
self.jsonSender.put('request/%s?status=%s' % (requestName, statusName))
result = self.jsonSender.get('request/%s' % requestName)
self.assertEqual(result[0]['RequestStatus'], statusName)
def testA_RequestPriority(self):
"""
_priorityChanges_
Do some fairly standard priority changes to the Request and
see how things react
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
result = self.jsonSender.put('request/testRequest', schema)
self.assertEqual(result[1], 200)
requestName = result[0]['RequestName']
workload = self.loadWorkload(requestName = requestName)
self.assertEqual(workload.priority(), 0)
# Set priority == 5
priority = 5
self.jsonSender.put('request/%s?priority=%s' % (requestName, priority))
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(request['RequestPriority'], priority)
workload = self.loadWorkload(requestName = requestName)
self.assertEqual(workload.priority(), priority)
# Set priority == 100
priority = 100
self.jsonSender.put('request/%s?priority=%s' % (requestName, priority))
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(request['RequestPriority'], priority)
workload = self.loadWorkload(requestName = requestName)
self.assertEqual(workload.priority(), priority)
# Set priority == -1
priority = -1
self.jsonSender.put('request/%s?priority=%s' % (requestName, priority))
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(request['RequestPriority'], priority)
workload = self.loadWorkload(requestName = requestName)
self.assertEqual(workload.priority(), priority)
# Let's move the request around a bit
self.changeStatusAndCheck(requestName = requestName,
statusName = 'testing-approved')
self.changeStatusAndCheck(requestName = requestName,
statusName = 'testing')
self.changeStatusAndCheck(requestName = requestName,
statusName = 'tested')
self.changeStatusAndCheck(requestName = requestName,
statusName = 'assignment-approved')
self.jsonSender.put(urllib.quote('assignment/%s/%s' % (teamName, requestName)))
# Set priority == 99
priority = 99
self.jsonSender.put('request/%s?priority=%s' % (requestName, priority))
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(request['RequestPriority'], priority)
workload = self.loadWorkload(requestName = requestName)
self.assertEqual(workload.priority(), priority)
def testB_InvalidPriority(self):
"""
_InvalidPriority_
Put in a bunch of invalid values for priorities and
see what the code makes of them.
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
result = self.jsonSender.put('request/testRequest', schema)
self.assertEqual(result[1], 200)
requestName = result[0]['RequestName']
raises = False
try:
priority = sys.maxint + 1
self.jsonSender.put('request/%s?priority=%s' % (requestName, priority))
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue("Priority must have abs() less then MAXINT!" in ex.result)
self.assertTrue(raises)
raises = False
try:
priority = -1 - sys.maxint
self.jsonSender.put('request/%s?priority=%s' % (requestName, priority))
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
print ex.result
self.assertTrue("Priority must have abs() less then MAXINT!" in ex.result)
class DBSBufferBlock(object):
"""
_DBSBufferBlock_
"""
def __init__(self, name, location, datasetpath):
"""
Just the necessary objects
Expects:
name: The blockname in full
location: The PNN of the site the block is at
"""
self.data = {
'dataset_conf_list': [], # List of dataset configurations
'file_conf_list': [], # List of files, the configuration for each
'files': [], # List of file objects
'block': {}, # Dict of block info
'processing_era': {}, # Dict of processing era info
'acquisition_era': {}, # Dict of acquisition era information
'primds': {}, # Dict of primary dataset info
'dataset': {}, # Dict of processed dataset info
'file_parent_list': [], # List of file parents
'dataset_parent_list':
[], # List of parent datasets (DBS requires this as list although it only allows one parent)
'close_settings': {}
} # Dict of info about block close settings
self.files = []
self.encoder = JSONRequests()
self.status = 'Open'
self.inBuff = False
self.startTime = time.time()
self.name = name
self.location = location
self.datasetpath = datasetpath
self.workflows = set()
self.data['block']['block_name'] = name
self.data['block']['origin_site_name'] = location
self.data['block']['open_for_writing'] = 1
self.data['block']['create_by'] = "WMAgent"
self.data['block']['creation_date'] = int(time.time())
self.data['block']['block_size'] = 0
self.data['block']['file_count'] = 0
self.data['block']['block_events'] = 0
self.data['close_settings'] = {}
self.data['close_settings']['block_close_max_wait_time'] = None
self.data['close_settings']['block_close_max_events'] = None
self.data['close_settings']['block_close_max_size'] = None
self.data['close_settings']['block_close_max_files'] = None
return
def encode(self):
"""
_encode_
Turn this into a JSON object for transmission
to DBS
"""
return self.encoder.encode(data=self.data)
def addFile(self, dbsFile, datasetType, primaryDatasetType):
"""
_addFile_
Add a DBSBufferFile object to our block
"""
if dbsFile['id'] in [x['id'] for x in self.files]:
msg = "Duplicate file inserted into DBSBufferBlock: %i\n" % (
dbsFile['id'])
msg += "Ignoring this file for now!\n"
logging.error(msg)
logging.debug("Block length: %i", len(self.files))
l = sorted([x['id'] for x in self.files])
logging.debug("First file: %s Last file: %s", l[0], l[-1])
return
for setting in self.data['close_settings']:
if self.data['close_settings'][setting] is None:
self.data['close_settings'][setting] = dbsFile[setting]
self.workflows.add(dbsFile['workflow'])
self.files.append(dbsFile)
self.data['block']['block_size'] += int(dbsFile['size'])
self.data['block']['file_count'] += 1
self.data['block']['block_events'] += int(dbsFile['events'])
# Assemble information for the file itself
fileDict = {}
fileDict['file_type'] = 'EDM'
fileDict['logical_file_name'] = dbsFile['lfn']
fileDict['file_size'] = dbsFile['size']
fileDict['event_count'] = dbsFile['events']
fileDict['last_modified_by'] = "WMAgent"
fileDict['last_modification_date'] = int(time.time())
fileDict['auto_cross_section'] = 0.0
# Do the checksums
for cktype in dbsFile['checksums']:
cksum = dbsFile['checksums'][cktype]
if cktype.lower() == 'cksum':
fileDict['check_sum'] = cksum
elif cktype.lower() == 'adler32':
fileDict['adler32'] = cksum
elif cktype.lower() == 'md5':
fileDict['md5'] = cksum
# Do the runs
lumiList = []
for run in dbsFile.getRuns():
for lumi in run.lumis:
dbsLumiDict = {'lumi_section_num': lumi, 'run_num': run.run}
if run.getEventsByLumi(lumi) is not None:
# if events is not None update event for dbs upload
dbsLumiDict['event_count'] = run.getEventsByLumi(lumi)
lumiList.append(dbsLumiDict)
fileDict['file_lumi_list'] = lumiList
# Append to the files list
self.data['files'].append(fileDict)
# If dataset_parent_list is defined don't add the file parentage.
# This means it is block from StepChain workflow and parentage of file will be resloved later
if not self.data['dataset_parent_list']:
# now add file to data
parentLFNs = dbsFile.getParentLFNs()
for lfn in parentLFNs:
self.addFileParent(child=dbsFile['lfn'], parent=lfn)
# Do the algo
algo = self.addConfiguration(release=dbsFile['appVer'],
psetHash=dbsFile['psetHash'],
appName=dbsFile['appName'],
outputLabel=dbsFile['appFam'],
globalTag=dbsFile['globalTag'])
# Now add the file with the algo
# Try to avoid messing with pointers here
fileAlgo = {}
fileAlgo.update(algo)
fileAlgo['lfn'] = dbsFile['lfn']
self.data['file_conf_list'].append(fileAlgo)
if dbsFile.get('acquisition_era', False):
self.setAcquisitionEra(dbsFile['acquisition_era'])
elif dbsFile.get('acquisitionEra', False):
self.setAcquisitionEra(dbsFile['acquisitionEra'])
if dbsFile.get('processingVer', False):
self.setProcessingVer(dbsFile['processingVer'])
elif dbsFile.get('processing_ver', False):
self.setProcessingVer(dbsFile['processing_ver'])
# Take care of the dataset
self.setDataset(datasetName=dbsFile['datasetPath'],
primaryType=primaryDatasetType,
datasetType=datasetType,
physicsGroup=dbsFile.get('physicsGroup', None),
prep_id=dbsFile.get('prep_id', None))
return
def addFileParent(self, child, parent):
"""
_addFileParent_
Add file parents to the data block
"""
info = {'parent_logical_file_name': parent, 'logical_file_name': child}
self.data['file_parent_list'].append(info)
return
def addBlockParent(self, parent):
"""
_addBlockParent_
Add the parents of the block
"""
self.data['block_parent_list'].append({'block_name': parent})
return
def addDatasetParent(self, parent):
"""
_addDatasetParent_
Add the parent datasets to the data block
"""
self.data['dataset_parent_list'].append(parent)
return
def setProcessingVer(self, procVer):
"""
_setProcessingVer_
Set the block's processing version.
"""
# compatibility statement for old style proc ver (still needed ?)
if procVer.count("-") == 1:
self.data["processing_era"]["processing_version"] = procVer.split(
"-v")[1]
else:
self.data["processing_era"]["processing_version"] = procVer
self.data["processing_era"]["create_by"] = "WMAgent"
self.data["processing_era"]["description"] = ""
return
def setAcquisitionEra(self, era, date=123456789):
"""
_setAcquisitionEra_
Set the acquisition era for the block
"""
self.data['acquisition_era']['acquisition_era_name'] = era
self.data['acquisition_era']['start_date'] = date
return
def setPhysicsGroup(self, group):
"""
_setPhysicsGroup_
Sets the name of the physics group to which the dataset is attached
"""
self.data['dataset']['physics_group_name'] = group
return
def getDatasetPath(self):
"""
_getDatasetPath_
Return the datasetpath
"""
return self.datasetpath
def getDataset(self):
"""
_getDataset_
Return the dataset (None if not set)
"""
return self.data['dataset'].get('dataset', None)
def setDataset(self,
datasetName,
primaryType,
datasetType,
physicsGroup=None,
prep_id=None,
overwrite=False):
"""
_setDataset_
Set all the information concerning a single dataset, including
the primary, processed and tier info
"""
if self.getDataset() != None and not overwrite:
# Do nothing, we already have a dataset
return
Lexicon.primaryDatasetType(primaryType)
if not datasetType in [
'VALID', 'PRODUCTION', 'INVALID', 'DEPRECATED', 'DELETED'
]:
msg = "Invalid processedDatasetType %s\n" % datasetType
logging.error(msg)
raise DBSBufferBlockException(msg)
try:
if datasetName[0] == '/':
_, primary, processed, tier = datasetName.split('/')
else:
primary, processed, tier = datasetName.split('/')
except Exception:
msg = "Invalid dataset name %s" % datasetName
logging.error(msg)
raise DBSBufferBlockException(msg)
# Do the primary dataset
self.data['primds']['primary_ds_name'] = primary
self.data['primds']['primary_ds_type'] = primaryType
self.data['primds']['create_by'] = "WMAgent"
self.data['primds']['creation_date'] = int(time.time())
# Do the processed
self.data['dataset']['physics_group_name'] = physicsGroup
self.data['dataset']['processed_ds_name'] = processed
self.data['dataset']['data_tier_name'] = tier
self.data['dataset']['dataset_access_type'] = datasetType
self.data['dataset']['dataset'] = datasetName
self.data['dataset']['prep_id'] = prep_id
# Add misc meta data.
self.data['dataset']['create_by'] = "WMAgent"
self.data['dataset']['last_modified_by'] = "WMAgent"
self.data['dataset']['creation_date'] = int(time.time())
self.data['dataset']['last_modification_date'] = int(time.time())
return
def addConfiguration(self,
release,
psetHash,
appName='cmsRun',
outputLabel='Merged',
globalTag='None'):
"""
_addConfiguration_
Add the algorithm config to the data block
"""
algo = {
'release_version': release,
'pset_hash': psetHash,
'app_name': appName,
'output_module_label': outputLabel,
'global_tag': globalTag
}
if not algo in self.data['dataset_conf_list']:
self.data['dataset_conf_list'].append(algo)
return algo
def getNFiles(self):
"""
_getNFiles_
Return the number of files in the block
"""
return len(self.files)
def getSize(self):
"""
_getSize_
Get size of block
"""
return self.data['block']['block_size']
def getNumEvents(self):
"""
_getNumEvents_
Get the number of events in the block
"""
return self.data['block']['block_events']
def getTime(self):
"""
_getTime_
Return the time the block has been running
"""
return time.time() - self.startTime
def getMaxBlockTime(self):
"""
_getMaxBlockTime_
Return the max time that the block should stay open
"""
return self.data['close_settings']['block_close_max_wait_time']
def getMaxBlockSize(self):
"""
_getMaxBlockSize_
Return the max size allowed for the block
"""
return self.data['close_settings']['block_close_max_size']
def getMaxBlockNumEvents(self):
"""
_getMaxBlockNumEvents_
Return the max number of events allowed for the block
"""
return self.data['close_settings']['block_close_max_events']
def getMaxBlockFiles(self):
"""
_getMaxBlockFiles_
Return the max number of files allowed for the block
"""
return self.data['close_settings']['block_close_max_files']
def getName(self):
"""
_getName_
Get Name
"""
return self.name
def getLocation(self):
"""
_getLocation_
Get location
"""
return self.location
def getStartTime(self):
"""
_getStartTime_
Get the time the block was opened at
"""
return self.startTime
def FillFromDBSBuffer(self, blockInfo):
"""
_FillFromDBSBuffer_
Take the info provided by LoadBlocks and
use it to create a block object
"""
# Blocks loaded out of the buffer should
# have both a creation time, and should
# be in the buffer (duh)
self.startTime = blockInfo.get('creation_date')
self.inBuff = True
if 'status' in blockInfo:
self.status = blockInfo['status']
if self.status == "Pending":
self.data['block']['open_for_writing'] = 0
del blockInfo['status']
for key in blockInfo:
self.data['block'][key] = blockInfo.get(key)
def convertToDBSBlock(self):
"""
convert to DBSBlock structure to upload to dbs
TODO: check file lumi event and validate event is not null
"""
block = {}
#TODO: instead of using key to remove need to change to keyToKeep
# Ask dbs team to publish the list (API)
keyToRemove = [
'insertedFiles', 'newFiles', 'file_count', 'block_size',
'origin_site_name', 'creation_date', 'open', 'Name',
'close_settings'
]
nestedKeyToRemove = [
'block.block_events', 'block.datasetpath', 'block.workflows'
]
dbsBufferToDBSBlockKey = {
'block_size': 'BlockSize',
'creation_date': 'CreationDate',
'file_count': 'NumberOfFiles',
'origin_site_name': 'location'
}
# clone the new DBSBlock dict after filtering out the data.
for key in self.data:
if key in keyToRemove:
continue
elif key in dbsBufferToDBSBlockKey:
block[dbsBufferToDBSBlockKey[key]] = copy.deepcopy(
self.data[key])
else:
block[key] = copy.deepcopy(self.data[key])
# delete nested key dictionary
for nestedKey in nestedKeyToRemove:
firstkey, subkey = nestedKey.split('.', 1)
if firstkey in block and subkey in block[firstkey]:
del block[firstkey][subkey]
return block
def setPendingAndCloseBlock(self):
"set the block status as Pending for upload as well as closed"
# Pending means ready to upload
self.status = "Pending"
# close block on DBS3 status
self.data['block']['open_for_writing'] = 0
class RequestManagerTest(RESTBaseUnitTest):
"""
Test RequestMgr Service client
It will start RequestMgr RESTService
Server DB is whatever env is set
This checks whether DS call makes without error and return the results.
This test only test service call returns without error.
The correctness of each function is tested in test/python/RequestManager_t/RequestMgr_t.py
"""
def initialize(self):
self.couchDBName = "reqmgr_t_0"
self.config = RequestManagerConfig(
'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel')
dbUrl = os.environ.get("DATABASE", None)
self.config.setDBUrl(dbUrl)
self.config.setFormatter('WMCore.WebTools.RESTFormatter')
self.config.setupRequestConfig()
self.config.setupCouchDatabase(dbName=self.couchDBName)
self.config.setPort(8899)
self.schemaModules = ["WMCore.RequestManager.RequestDB"]
def setUp(self):
RESTBaseUnitTest.setUp(self)
self.testInit.setupCouch("%s" % self.couchDBName, "GroupUser",
"ConfigCache", "ReqMgr")
self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats")
# logging stuff from TestInit is broken, setting myself
l = logging.getLogger()
l.setLevel(logging.DEBUG)
self.params = {}
self.params['endpoint'] = self.config.getServerUrl()
self.reqService = RequestManagerDS(self.params)
self.jsonSender = JSONRequests(self.config.getServerUrl())
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName=userName,
groupName=groupName,
teamName=teamName)
schema['ConfigCacheID'] = self.createConfig()
schema['CouchDBName'] = self.couchDBName
schema['CouchWorkloadDBName'] = self.couchDBName
try:
r = self.jsonSender.put('request', schema)
try:
self.requestName = r[0]['RequestName']
except:
self.requestName = r[0].values()[0]['RequestName']
except Exception as ex:
msg = traceback.format_exc()
print "Exception during set up, reason: %s" % msg
raise ex
def tearDown(self):
self.config.deleteWorkloadCache()
RESTBaseUnitTest.tearDown(self)
self.testInit.tearDownCouch()
def createConfig(self, bad=False):
"""
_createConfig_
Create a config of some sort that we can load out of ConfigCache
"""
PSetTweak = {
'process': {
'outputModules_': ['ThisIsAName'],
'ThisIsAName': {
'dataset': {
'dataTier': 'RECO',
'filterName': 'Filter'
}
}
}
}
BadTweak = {
'process': {
'outputModules_': ['ThisIsAName1', 'ThisIsAName2'],
'ThisIsAName1': {
'dataset': {
'dataTier': 'RECO',
'filterName': 'Filter'
}
},
'ThisIsAName2': {
'dataset': {
'dataTier': 'RECO',
'filterName': 'Filter'
}
}
}
}
configCache = ConfigCache(os.environ["COUCHURL"],
couchDBName=self.couchDBName)
configCache.createUserGroup(groupname="testGroup", username='******')
if bad:
configCache.setPSetTweaks(PSetTweak=BadTweak)
else:
configCache.setPSetTweaks(PSetTweak=PSetTweak)
configCache.save()
return configCache.getCouchID()
@attr("integration")
def testA_RequestManagerService(self):
requestName = self.requestName
request = self.reqService.getRequest(requestName)
# minimal test : it's return type and the some value inside
self.assertEqual(type(request), dict)
self.assertTrue(len(request) > 0)
# Test putTeam
self.reqService.putTeam("team_usa")
self.assertTrue('team_usa' in self.jsonSender.get('team')[0])
self.jsonSender.put('assignment/%s/%s' % ("team_usa", requestName))
request = self.reqService.getAssignment(teamName="team_usa")
self.assertEqual(type(request), list)
self.assertTrue(len(request) > 0)
request = self.reqService.getAssignment(request=requestName)
self.assertEqual(type(request), list)
self.assertTrue(len(request) > 0)
self.reqService.sendMessage(requestName, "error")
self.reqService.putWorkQueue(requestName, "http://test_url")
self.reqService.reportRequestProgress(requestName,
percent_complete=100,
percent_success=90)
self.reqService.updateRequestStatus(requestName, "running-open")