def execute(self, *args):
dag_status = int(args[0])
failed_count = int(args[1])
restinstance = args[2]
resturl = args[3]
if dag_status in [1, 2, 3]:
if dag_status == 3:
msg = "Task aborted because the maximum number of failures was hit; %d total failed jobs." % failed_count
else:
msg = "Task failed overall; %d failed jobs" % failed_count
configreq = {'workflow': kw['task']['tm_taskname'],
'substatus': "FAILED",
'subfailure': base64.b64encode(str(e)),}
data = urllib.urlencode(configreq)
server = HTTPRequests(restinstance, os.environ['X509_USER_PROXY'], os.environ['X509_USER_PROXY'])
server.put(resturl, data=data)
return dag_status
def execute(self, *args):
dag_status = int(args[0])
failed_count = int(args[1])
restinstance = args[2]
resturl = args[3]
if dag_status in [1, 2, 3]:
if dag_status == 3:
msg = "Task aborted because the maximum number of failures was hit; %d total failed jobs." % failed_count
else:
msg = "Task failed overall; %d failed jobs" % failed_count
configreq = {
'workflow': kw['task']['tm_taskname'],
'substatus': "FAILED",
'subfailure': base64.b64encode(str(e)),
}
data = urllib.urlencode(configreq)
server = HTTPRequests(restinstance, os.environ['X509_USER_PROXY'],
os.environ['X509_USER_PROXY'])
server.put(resturl, data=data)
return dag_status
class FileTransfersTest(unittest.TestCase):
"""
_DashboardAPITest_
Unit tests for the FileTransfers API
"""
def setUp(self):
"""
Setup for unit tests
"""
self.server = HTTPRequests(os.environ['SERVER_HOST'],
os.environ['X509_USER_PROXY'],
os.environ['X509_USER_PROXY'])
self.lfnBase = '/store/temp/user/%s/my_cool_dataset-%s/file-%s-%s.root'
self.fileDoc = {
'id': 'OVERWRITE',
'username': '******',
'taskname': 'OVERWRITE',
'start_time': 0,
'destination': 'T2_CH_CERN',
'destination_lfn': 'OVERWRITE',
'source': 'T2_US_Caltech',
'source_lfn': 'OVERWRITE',
'filesize': random.randint(1, 9999),
'publish': 1,
'transfer_state': 'OVERWRITE',
'publication_state': 'OVERWRITE',
'job_id': 1,
'job_retry_count': 0,
'type': 'log',
'rest_host': 'cmsweb.cern.ch',
'rest_uri': '/crabserver/prod/'
}
self.ids = []
self.users = [
'jbalcas', 'mmascher', 'dciangot', 'riahi', 'erupeika', 'sbelforte'
] # just random users for tests
self.tasks = {}
self.totalFiles = 10
def testFileTransferPUT(self):
"""
_testFileTransferPUT_
Just test simple testFileTransferPUT with fake data
"""
# We just sent fake data which is not monitored by dashboard.
# Also only the first time to decide is publication ON or NOT
for user in self.users:
timestamp = time.strftime('%y%m%d_%H%M%S', time.gmtime())
for i in range(self.totalFiles):
now = int(time.time())
# Generate a taskname
workflowName = ""
taskname = ""
if user not in self.tasks:
workflowName = "".join([
random.choice(string.ascii_lowercase)
for _ in range(20)
]) + "_" + str(now)
publicationState = random.choice(['NEW', 'NOT_REQUIRED'])
else:
workflowName = self.tasks[user]['workflowName']
publicationState = self.tasks[user]['publication']
transferState = random.choice(['NEW', 'DONE'])
taskname = generateTaskName(user, workflowName, timestamp)
finalLfn = self.lfnBase % (user, workflowName, i,
random.randint(1, 9999))
idHash = getHashLfn(finalLfn)
self.fileDoc['id'] = idHash
self.fileDoc['job_id'] = i
self.fileDoc['username'] = user
self.fileDoc['taskname'] = taskname
self.fileDoc['start_time'] = int(time.time())
self.fileDoc['source_lfn'] = finalLfn
self.fileDoc['destination_lfn'] = finalLfn
self.fileDoc['transfer_state'] = transferState
self.fileDoc['publication_state'] = publicationState
print(self.fileDoc)
self.server.put('/crabserver/dev/fileusertransfers',
data=encodeRequest(self.fileDoc))
# if I will put the same doc twice, it should raise an error.
# self.server.put('/crabserver/dev/fileusertransfers', data=urllib.urlencode(self.fileDoc))
# This tasks are for the future and next calls
if user not in self.tasks:
self.tasks[user] = {
'workflowName': workflowName,
'taskname': taskname,
'listOfIds': [],
'publication': publicationState,
'toTransfer': 0,
'toPublish': 0,
'total': self.totalFiles
}
if self.tasks[user]['publication'] == 'NEW':
self.tasks[user]['toPublish'] += 1
if transferState == 'NEW':
self.tasks[user]['toTransfer'] += 1
self.tasks[user]['listOfIds'].append(idHash)
# This should raise an error
for username in self.tasks:
taskname = self.tasks[username]['taskname']
for query in ['getTransferStatus', 'getPublicationStatus']:
result = self.server.get('/crabserver/dev/fileusertransfers',
data=encodeRequest({
'subresource':
query,
'username':
username,
'taskname':
taskname
}))
print(result)
print(result[0]['result'])
taskInfoDict = oracleOutputMapping(result, 'id')
print(taskInfoDict)
for key, docDict in taskInfoDict.items():
result = self.server.get(
'/crabserver/dev/fileusertransfers',
data=encodeRequest({
'subresource': 'getById',
'id': key
}))
randomUsers = random.sample(
set(self.users), 3
) # Take half of the users and kill their transfers for specific task
for username in randomUsers:
taskname = self.tasks[username]['taskname']
result = self.server.post('/crabserver/dev/fileusertransfers',
data=encodeRequest({
'subresource': 'killTransfers',
'username': username,
'taskname': taskname
}))
print(result)
# oneUser is left for killing a list of IDs
# leftUsers will be killing transfers one by one for specific id.
leftUsers = list(set(self.users) - set(randomUsers))
oneUser = random.sample(set(leftUsers), 1)
leftUsers = list(set(leftUsers) - set(oneUser))
for username in leftUsers:
# First get all left ids for this users
result = self.server.get('/crabserver/dev/fileusertransfers',
data=encodeRequest({
'subresource':
'getTransferStatus',
'username':
username,
'taskname':
self.tasks[username]['taskname']
}))
resultOut = oracleOutputMapping(result, None)
print("**" * 50)
for outDict in resultOut:
print(outDict)
result = self.server.post('/crabserver/dev/fileusertransfers',
data=encodeRequest({
'subresource':
'killTransfersById',
'username':
username,
'listOfIds':
outDict['id']
}))
print(result)
print(resultOut)
print(result)
for username in oneUser:
result = self.server.post(
'/crabserver/dev/fileusertransfers',
data=encodeRequest(
{
'subresource': 'killTransfersById',
'username': username,
'listOfIds': self.tasks[username]['listOfIds']
}, ['listOfIds']))
# As it asks to kill all which are in new, need to double check what we submitted before and if the output of killed is correct
print(result)
print(self.tasks[username])
def __call__(self):
valid = False
configmsg = 'Default'
self.logger.debug("Started submission")
# Get some debug parameters
oneEventMode = hasattr(self.configuration, 'Debug') and \
getattr(self.configuration.Debug, 'oneEventMode')
######### Check if the user provided unexpected parameters ########
#init the dictionary with all the known parameters
SpellChecker.DICTIONARY = SpellChecker.train( [ val['config'] for _, val in self.requestmapper.iteritems() if val['config'] ] + \
[ x for x in self.otherConfigParams ] )
#iterate on the parameters provided by the user
for section in self.configuration.listSections_():
for attr in getattr(self.configuration, section).listSections_():
par = (section + '.' + attr)
#if the parameter is not know exit, but try to correct it before
if not SpellChecker.is_correct( par ):
msg = 'The parameter %s is not known.' % par
msg += '' if SpellChecker.correct(par) == par else ' Did you mean %s?' % SpellChecker.correct(par)
raise ConfigurationException(msg)
#usertarball and cmsswconfig use this parameter and we should set it up in a correct way
self.configuration.General.serverUrl = self.serverurl
uniquerequestname = None
self.logger.debug("Working on %s" % str(self.requestarea))
configreq = {}
for param in self.requestmapper:
mustbetype = getattr(types, self.requestmapper[param]['type'])
if self.requestmapper[param]['config']:
attrs = self.requestmapper[param]['config'].split('.')
temp = self.configuration
for attr in attrs:
temp = getattr(temp, attr, None)
if temp is None:
break
if temp is not None:
if mustbetype == type(temp):
configreq[param] = temp
else:
raise ConfigurationException("Invalid type " + str(type(temp)) + " for parameter " + self.requestmapper[param]['config'] \
+ ". It is needed a " + str(mustbetype) + ".")
elif self.requestmapper[param]['default'] is not None:
configreq[param] = self.requestmapper[param]['default']
temp = self.requestmapper[param]['default']
elif self.requestmapper[param]['required']:
raise ConfigurationException("Missing parameter " + self.requestmapper[param]['config'] + " from the configuration.")
else:
## parameter not strictly required
pass
if param == "workflow":
if mustbetype == type(self.requestname):
configreq["workflow"] = self.requestname
elif param in ['savelogsflag','publication','nonprodsw','ignorelocality','saveoutput']:#TODO use clientmappig to do this
configreq[param] = 1 if temp else 0
elif param in ['dbsurl','publishdbsurl']:
if param == 'dbsurl':
dbstype = 'reader'
elif param == 'publishdbsurl':
dbstype = 'writer'
alloweddbsurls = DBSURLS[dbstype].values()
alloweddbsurlsaliases = DBSURLS[dbstype].keys()
if configreq[param] in alloweddbsurlsaliases:
configreq[param] = DBSURLS[dbstype][configreq[param]]
else:
if configreq[param].rstrip('/') in alloweddbsurls:
configreq[param] = configreq[param].rstrip('/')
else:
raise ConfigurationException("Invalid argument " + configreq[param] + " for parameter " + self.requestmapper[param]['config'] + " in the configuration.")
if (configreq['saveoutput'] or configreq['savelogsflag']) and 'asyncdest' not in configreq:
raise ConfigurationException("Missing parameter " + self.requestmapper['asyncdest']['config'] + " from the configuration.")
# Add debug parameters to the configreq dict
configreq['oneEventMode'] = int(oneEventMode)
jobconfig = {}
self.configuration.JobType.proxyfilename = self.proxyfilename
self.configuration.JobType.capath = HTTPRequests.getCACertPath()
#get the backend URLs from the server external configuration
serverBackendURLs = server_info('backendurls', self.serverurl, self.proxyfilename, self.getUrl(self.instance, resource='info'))
#if cacheSSL is specified in the server external configuration we will use it to upload the sandbox (baseURL will be ignored)
self.configuration.JobType.filecacheurl = serverBackendURLs['cacheSSL'] if 'cacheSSL' in serverBackendURLs else None
pluginParams = [ self.configuration, self.logger, os.path.join(self.requestarea, 'inputs') ]
if getattr(self.configuration.JobType, 'pluginName', None) is not None:
jobtypes = getJobTypes()
plugjobtype = jobtypes[upper(self.configuration.JobType.pluginName)](*pluginParams)
inputfiles, jobconfig, isbchecksum = plugjobtype.run(configreq)
else:
fullname = self.configuration.JobType.externalPluginFile
basename = os.path.basename(fullname).split('.')[0]
plugin = addPlugin(fullname)[basename]
pluginInst = plugin(*pluginParams)
inputfiles, jobconfig, isbchecksum = pluginInst.run(configreq)
if not configreq['publishname']:
configreq['publishname'] = isbchecksum
else:
configreq['publishname'] = "%s-%s" %(configreq['publishname'], isbchecksum)
configreq.update(jobconfig)
server = HTTPRequests(self.serverurl, self.proxyfilename, self.proxyfilename, version=__version__)
self.logger.info("Sending the request to the server")
self.logger.debug("Submitting %s " % str( configreq ) )
dictresult, status, reason = server.put( self.uri, data = self._encodeRequest(configreq) )
self.logger.debug("Result: %s" % dictresult)
if status != 200:
msg = "Problem sending the request:\ninput:%s\noutput:%s\nreason:%s" % (str(configreq), str(dictresult), str(reason))
raise RESTCommunicationException(msg)
elif dictresult.has_key("result"):
uniquerequestname = dictresult["result"][0]["RequestName"]
else:
msg = "Problem during submission, no request ID returned:\ninput:%s\noutput:%s\nreason:%s" \
% (str(configreq), str(dictresult), str(reason))
raise RESTCommunicationException(msg)
tmpsplit = self.serverurl.split(':')
createCache(self.requestarea, tmpsplit[0], tmpsplit[1] if len(tmpsplit)>1 else '', uniquerequestname,
voRole=self.voRole, voGroup=self.voGroup, instance=self.instance,
originalConfig = self.configuration)
self.logger.info("%sSuccess%s: Your task has been delivered to the CRAB3 server." %(colors.GREEN, colors.NORMAL))
if not self.options.wait:
self.logger.info("Please use 'crab status' to check how the submission process proceed")
self.logger.debug("Request ID: %s " % uniquerequestname)
if self.options.wait:
self.checkStatusLoop(server,uniquerequestname)
return uniquerequestname
class FileTransfersTest(unittest.TestCase):
"""
_DashboardAPITest_
Unit tests for the FileTransfers API
"""
def setUp(self):
"""
Setup for unit tests
"""
self.server = HTTPRequests(os.environ['SERVER_HOST'], os.environ['X509_USER_PROXY'], os.environ['X509_USER_PROXY'])
self.lfnBase = '/store/temp/user/%s/my_cool_dataset-%s/file-%s-%s.root'
self.fileDoc = {'id': 'OVERWRITE',
'username': '******',
'taskname': 'OVERWRITE',
'start_time': 0,
'destination': 'T2_CH_CERN',
'destination_lfn': 'OVERWRITE',
'source': 'T2_US_Caltech',
'source_lfn': 'OVERWRITE',
'filesize': random.randint(1, 9999),
'publish': 1,
'transfer_state': 'OVERWRITE',
'publication_state': 'OVERWRITE',
'job_id': 1,
'job_retry_count': 0,
'type': 'log',
'rest_host': 'cmsweb.cern.ch',
'rest_uri': '/crabserver/prod/'}
self.ids = []
self.users = ['jbalcas', 'mmascher', 'dciangot', 'riahi', 'erupeika', 'sbelforte'] # just random users for tests
self.tasks = {}
self.totalFiles = 10
def testFileTransferPUT(self):
"""
_testFileTransferPUT_
Just test simple testFileTransferPUT with fake data
"""
# We just sent fake data which is not monitored by dashboard.
# Also only the first time to decide is publication ON or NOT
for user in self.users:
timestamp = time.strftime('%y%m%d_%H%M%S', time.gmtime())
for i in range(self.totalFiles):
now = int(time.time())
# Generate a taskname
workflowName = ""
taskname = ""
if user not in self.tasks:
workflowName = "".join([random.choice(string.ascii_lowercase) for _ in range(20)]) + "_" + str(now)
publicationState = random.choice(['NEW', 'NOT_REQUIRED'])
else:
workflowName = self.tasks[user]['workflowName']
publicationState = self.tasks[user]['publication']
transferState = random.choice(['NEW', 'DONE'])
taskname = generateTaskName(user, workflowName, timestamp)
finalLfn = self.lfnBase % (user, workflowName, i, random.randint(1, 9999))
idHash = getHashLfn(finalLfn)
self.fileDoc['id'] = idHash
self.fileDoc['job_id'] = i
self.fileDoc['username'] = user
self.fileDoc['taskname'] = taskname
self.fileDoc['start_time'] = int(time.time())
self.fileDoc['source_lfn'] = finalLfn
self.fileDoc['destination_lfn'] = finalLfn
self.fileDoc['transfer_state'] = transferState
self.fileDoc['publication_state'] = publicationState
print(self.fileDoc)
self.server.put('/crabserver/dev/fileusertransfers', data=encodeRequest(self.fileDoc))
# if I will put the same doc twice, it should raise an error.
# self.server.put('/crabserver/dev/fileusertransfers', data=urllib.urlencode(self.fileDoc))
# This tasks are for the future and next calls
if user not in self.tasks:
self.tasks[user] = {'workflowName': workflowName, 'taskname': taskname, 'listOfIds': [],
'publication': publicationState, 'toTransfer': 0, 'toPublish': 0, 'total': self.totalFiles}
if self.tasks[user]['publication'] == 'NEW':
self.tasks[user]['toPublish'] += 1
if transferState == 'NEW':
self.tasks[user]['toTransfer'] += 1
self.tasks[user]['listOfIds'].append(idHash)
# This should raise an error
for username in self.tasks:
taskname = self.tasks[username]['taskname']
for query in ['getTransferStatus', 'getPublicationStatus']:
result = self.server.get('/crabserver/dev/fileusertransfers', data=encodeRequest({'subresource': query,
'username': username,
'taskname': taskname}))
print(result)
print(result[0]['result'])
taskInfoDict = oracleOutputMapping(result, 'id')
print(taskInfoDict)
for key, docDict in taskInfoDict.items():
result = self.server.get('/crabserver/dev/fileusertransfers', data=encodeRequest({'subresource': 'getById', 'id': key}))
randomUsers = random.sample(set(self.users), 3) # Take half of the users and kill their transfers for specific task
for username in randomUsers:
taskname = self.tasks[username]['taskname']
result = self.server.post('/crabserver/dev/fileusertransfers', data=encodeRequest({'subresource': 'killTransfers',
'username': username,
'taskname': taskname}))
print(result)
# oneUser is left for killing a list of IDs
# leftUsers will be killing transfers one by one for specific id.
leftUsers = list(set(self.users) - set(randomUsers))
oneUser = random.sample(set(leftUsers), 1)
leftUsers = list(set(leftUsers) - set(oneUser))
for username in leftUsers:
# First get all left ids for this users
result = self.server.get('/crabserver/dev/fileusertransfers', data=encodeRequest({'subresource': 'getTransferStatus',
'username': username,
'taskname': self.tasks[username]['taskname']}))
resultOut = oracleOutputMapping(result, None)
print("**"*50)
for outDict in resultOut:
print(outDict)
result = self.server.post('/crabserver/dev/fileusertransfers', data=encodeRequest({'subresource': 'killTransfersById',
'username': username,
'listOfIds': outDict['id']}))
print(result)
print(resultOut)
print(result)
for username in oneUser:
result = self.server.post('/crabserver/dev/fileusertransfers', data=encodeRequest({'subresource': 'killTransfersById',
'username': username,
'listOfIds': self.tasks[username]['listOfIds']}, ['listOfIds']))
# As it asks to kill all which are in new, need to double check what we submitted before and if the output of killed is correct
print(result)
print(self.tasks[username])
def __call__(self):
valid = False
configmsg = 'Default'
if not os.path.isfile(self.options.config):
raise MissingOptionException("Configuration file '%s' not found" % self.options.config)
self.logger.debug("Started submission")
######### Check if the user provided unexpected parameters ########
#init the dictionary with all the known parameters
SpellChecker.DICTIONARY = SpellChecker.train( [ val['config'] for _, val in self.requestmapper.iteritems() if val['config'] ] + \
[ x for x in self.otherConfigParams ] )
#iterate on the parameters provided by the user
for section in self.configuration.listSections_():
for attr in getattr(self.configuration, section).listSections_():
par = (section + '.' + attr)
#if the parameter is not know exit, but try to correct it before
if not SpellChecker.is_correct( par ):
msg = 'The parameter %s is not known.' % par
msg += '' if SpellChecker.correct(par) == par else ' Did you mean %s?' % SpellChecker.correct(par)
raise ConfigurationException(msg)
#usertarball and cmsswconfig use this parameter and we should set it up in a correct way
self.configuration.General.serverUrl = self.serverurl
uniquerequestname = None
self.logger.debug("Working on %s" % str(self.requestarea))
configreq = {}
for param in self.requestmapper:
mustbetype = getattr(types, self.requestmapper[param]['type'])
if self.requestmapper[param]['config']:
attrs = self.requestmapper[param]['config'].split('.')
temp = self.configuration
for attr in attrs:
temp = getattr(temp, attr, None)
if temp is None:
break
if temp:
if mustbetype == type(temp):
configreq[param] = temp
else:
raise ConfigurationException("Invalid type " + str(type(temp)) + " for parameter " + self.requestmapper[param]['config'] \
+ ". It is needed a " + str(mustbetype) + ".")
elif self.requestmapper[param]['default'] is not None:
configreq[param] = self.requestmapper[param]['default']
elif self.requestmapper[param]['required']:
raise ConfigurationException("Missing parameter " + self.requestmapper[param]['config'] + " from the configuration.")
else:
## parameter not strictly required
pass
if param == "workflow":
if mustbetype == type(self.requestname):
configreq["workflow"] = self.requestname
elif param == "savelogsflag":
configreq["savelogsflag"] = 1 if temp else 0
elif param == "publication":
configreq["publication"] = 1 if temp else 0
elif param == "blacklistT1":
blacklistT1 = self.voRole != 't1access'
#if the user choose to remove the automatic T1 blacklisting and has not the t1acces role
if getattr (self.configuration.Site, 'removeT1Blacklisting', False) and blacklistT1:
self.logger.info("WARNING: You disabled the T1 automatic blacklisting without having the t1access role")
blacklistT1 = False
configreq["blacklistT1"] = 1 if blacklistT1 else 0
jobconfig = {}
self.configuration.JobType.proxyfilename = self.proxyfilename
self.configuration.JobType.capath = HTTPRequests.getCACertPath()
#get the backend URLs from the server external configuration
serverBackendURLs = server_info('backendurls', self.serverurl, self.proxyfilename, self.getUrl(self.instance, resource='info'))
#if cacheSSL is specified in the server external configuration we will use it to upload the sandbox (baseURL will be ignored)
self.configuration.JobType.filecacheurl = serverBackendURLs['cacheSSL'] if 'cacheSSL' in serverBackendURLs else None
#otherwise we will contact the baseurl to get the cache hostname
self.configuration.JobType.baseurl = serverBackendURLs['baseURL']
pluginParams = [ self.configuration, self.logger, os.path.join(self.requestarea, 'inputs') ]
if getattr(self.configuration.JobType, 'pluginName', None) is not None:
jobtypes = getJobTypes()
plugjobtype = jobtypes[upper(self.configuration.JobType.pluginName)](*pluginParams)
inputfiles, jobconfig, isbchecksum = plugjobtype.run(configreq)
else:
fullname = self.configuration.JobType.externalPluginFile
basename = os.path.basename(fullname).split('.')[0]
plugin = addPlugin(fullname)[basename]
pluginInst = plugin(*pluginParams)
inputfiles, jobconfig, isbchecksum = pluginInst.run(configreq)
if not configreq['publishname']:
configreq['publishname'] = isbchecksum
else:
configreq['publishname'] = "%s-%s" %(configreq['publishname'], isbchecksum)
configreq.update(jobconfig)
server = HTTPRequests(self.serverurl, self.proxyfilename, self.proxyfilename, version=__version__)
self.logger.info("Sending the request to the server")
self.logger.debug("Submitting %s " % str( configreq ) )
dictresult, status, reason = server.put( self.uri, data = self._encodeRequest(configreq) )
self.logger.debug("Result: %s" % dictresult)
if status != 200:
msg = "Problem sending the request:\ninput:%s\noutput:%s\nreason:%s" % (str(configreq), str(dictresult), str(reason))
raise RESTCommunicationException(msg)
elif dictresult.has_key("result"):
uniquerequestname = dictresult["result"][0]["RequestName"]
else:
msg = "Problem during submission, no request ID returned:\ninput:%s\noutput:%s\nreason:%s" \
% (str(configreq), str(dictresult), str(reason))
raise RESTCommunicationException(msg)
tmpsplit = self.serverurl.split(':')
createCache(self.requestarea, tmpsplit[0], tmpsplit[1] if len(tmpsplit)>1 else '', uniquerequestname,
voRole=self.voRole, voGroup=self.voGroup, instance=self.instance)
self.logger.info("Submission completed")
self.logger.debug("Request ID: %s " % uniquerequestname)
self.logger.debug("Ended submission")
return uniquerequestname
