def execute(self, *args): dag_status = int(args[0]) failed_count = int(args[1]) restinstance = args[2] resturl = args[3] if dag_status in [1, 2, 3]: if dag_status == 3: msg = "Task aborted because the maximum number of failures was hit; %d total failed jobs." % failed_count else: msg = "Task failed overall; %d failed jobs" % failed_count configreq = {'workflow': kw['task']['tm_taskname'], 'substatus': "FAILED", 'subfailure': base64.b64encode(str(e)),} data = urllib.urlencode(configreq) server = HTTPRequests(restinstance, os.environ['X509_USER_PROXY'], os.environ['X509_USER_PROXY']) server.put(resturl, data=data) return dag_status
def execute(self, *args): dag_status = int(args[0]) failed_count = int(args[1]) restinstance = args[2] resturl = args[3] if dag_status in [1, 2, 3]: if dag_status == 3: msg = "Task aborted because the maximum number of failures was hit; %d total failed jobs." % failed_count else: msg = "Task failed overall; %d failed jobs" % failed_count configreq = { 'workflow': kw['task']['tm_taskname'], 'substatus': "FAILED", 'subfailure': base64.b64encode(str(e)), } data = urllib.urlencode(configreq) server = HTTPRequests(restinstance, os.environ['X509_USER_PROXY'], os.environ['X509_USER_PROXY']) server.put(resturl, data=data) return dag_status
class FileTransfersTest(unittest.TestCase): """ _DashboardAPITest_ Unit tests for the FileTransfers API """ def setUp(self): """ Setup for unit tests """ self.server = HTTPRequests(os.environ['SERVER_HOST'], os.environ['X509_USER_PROXY'], os.environ['X509_USER_PROXY']) self.lfnBase = '/store/temp/user/%s/my_cool_dataset-%s/file-%s-%s.root' self.fileDoc = { 'id': 'OVERWRITE', 'username': '******', 'taskname': 'OVERWRITE', 'start_time': 0, 'destination': 'T2_CH_CERN', 'destination_lfn': 'OVERWRITE', 'source': 'T2_US_Caltech', 'source_lfn': 'OVERWRITE', 'filesize': random.randint(1, 9999), 'publish': 1, 'transfer_state': 'OVERWRITE', 'publication_state': 'OVERWRITE', 'job_id': 1, 'job_retry_count': 0, 'type': 'log', 'rest_host': 'cmsweb.cern.ch', 'rest_uri': '/crabserver/prod/' } self.ids = [] self.users = [ 'jbalcas', 'mmascher', 'dciangot', 'riahi', 'erupeika', 'sbelforte' ] # just random users for tests self.tasks = {} self.totalFiles = 10 def testFileTransferPUT(self): """ _testFileTransferPUT_ Just test simple testFileTransferPUT with fake data """ # We just sent fake data which is not monitored by dashboard. # Also only the first time to decide is publication ON or NOT for user in self.users: timestamp = time.strftime('%y%m%d_%H%M%S', time.gmtime()) for i in range(self.totalFiles): now = int(time.time()) # Generate a taskname workflowName = "" taskname = "" if user not in self.tasks: workflowName = "".join([ random.choice(string.ascii_lowercase) for _ in range(20) ]) + "_" + str(now) publicationState = random.choice(['NEW', 'NOT_REQUIRED']) else: workflowName = self.tasks[user]['workflowName'] publicationState = self.tasks[user]['publication'] transferState = random.choice(['NEW', 'DONE']) taskname = generateTaskName(user, workflowName, timestamp) finalLfn = self.lfnBase % (user, workflowName, i, random.randint(1, 9999)) idHash = getHashLfn(finalLfn) self.fileDoc['id'] = idHash self.fileDoc['job_id'] = i self.fileDoc['username'] = user self.fileDoc['taskname'] = taskname self.fileDoc['start_time'] = int(time.time()) self.fileDoc['source_lfn'] = finalLfn self.fileDoc['destination_lfn'] = finalLfn self.fileDoc['transfer_state'] = transferState self.fileDoc['publication_state'] = publicationState print(self.fileDoc) self.server.put('/crabserver/dev/fileusertransfers', data=encodeRequest(self.fileDoc)) # if I will put the same doc twice, it should raise an error. # self.server.put('/crabserver/dev/fileusertransfers', data=urllib.urlencode(self.fileDoc)) # This tasks are for the future and next calls if user not in self.tasks: self.tasks[user] = { 'workflowName': workflowName, 'taskname': taskname, 'listOfIds': [], 'publication': publicationState, 'toTransfer': 0, 'toPublish': 0, 'total': self.totalFiles } if self.tasks[user]['publication'] == 'NEW': self.tasks[user]['toPublish'] += 1 if transferState == 'NEW': self.tasks[user]['toTransfer'] += 1 self.tasks[user]['listOfIds'].append(idHash) # This should raise an error for username in self.tasks: taskname = self.tasks[username]['taskname'] for query in ['getTransferStatus', 'getPublicationStatus']: result = self.server.get('/crabserver/dev/fileusertransfers', data=encodeRequest({ 'subresource': query, 'username': username, 'taskname': taskname })) print(result) print(result[0]['result']) taskInfoDict = oracleOutputMapping(result, 'id') print(taskInfoDict) for key, docDict in taskInfoDict.items(): result = self.server.get( '/crabserver/dev/fileusertransfers', data=encodeRequest({ 'subresource': 'getById', 'id': key })) randomUsers = random.sample( set(self.users), 3 ) # Take half of the users and kill their transfers for specific task for username in randomUsers: taskname = self.tasks[username]['taskname'] result = self.server.post('/crabserver/dev/fileusertransfers', data=encodeRequest({ 'subresource': 'killTransfers', 'username': username, 'taskname': taskname })) print(result) # oneUser is left for killing a list of IDs # leftUsers will be killing transfers one by one for specific id. leftUsers = list(set(self.users) - set(randomUsers)) oneUser = random.sample(set(leftUsers), 1) leftUsers = list(set(leftUsers) - set(oneUser)) for username in leftUsers: # First get all left ids for this users result = self.server.get('/crabserver/dev/fileusertransfers', data=encodeRequest({ 'subresource': 'getTransferStatus', 'username': username, 'taskname': self.tasks[username]['taskname'] })) resultOut = oracleOutputMapping(result, None) print("**" * 50) for outDict in resultOut: print(outDict) result = self.server.post('/crabserver/dev/fileusertransfers', data=encodeRequest({ 'subresource': 'killTransfersById', 'username': username, 'listOfIds': outDict['id'] })) print(result) print(resultOut) print(result) for username in oneUser: result = self.server.post( '/crabserver/dev/fileusertransfers', data=encodeRequest( { 'subresource': 'killTransfersById', 'username': username, 'listOfIds': self.tasks[username]['listOfIds'] }, ['listOfIds'])) # As it asks to kill all which are in new, need to double check what we submitted before and if the output of killed is correct print(result) print(self.tasks[username])
def __call__(self): valid = False configmsg = 'Default' self.logger.debug("Started submission") # Get some debug parameters oneEventMode = hasattr(self.configuration, 'Debug') and \ getattr(self.configuration.Debug, 'oneEventMode') ######### Check if the user provided unexpected parameters ######## #init the dictionary with all the known parameters SpellChecker.DICTIONARY = SpellChecker.train( [ val['config'] for _, val in self.requestmapper.iteritems() if val['config'] ] + \ [ x for x in self.otherConfigParams ] ) #iterate on the parameters provided by the user for section in self.configuration.listSections_(): for attr in getattr(self.configuration, section).listSections_(): par = (section + '.' + attr) #if the parameter is not know exit, but try to correct it before if not SpellChecker.is_correct( par ): msg = 'The parameter %s is not known.' % par msg += '' if SpellChecker.correct(par) == par else ' Did you mean %s?' % SpellChecker.correct(par) raise ConfigurationException(msg) #usertarball and cmsswconfig use this parameter and we should set it up in a correct way self.configuration.General.serverUrl = self.serverurl uniquerequestname = None self.logger.debug("Working on %s" % str(self.requestarea)) configreq = {} for param in self.requestmapper: mustbetype = getattr(types, self.requestmapper[param]['type']) if self.requestmapper[param]['config']: attrs = self.requestmapper[param]['config'].split('.') temp = self.configuration for attr in attrs: temp = getattr(temp, attr, None) if temp is None: break if temp is not None: if mustbetype == type(temp): configreq[param] = temp else: raise ConfigurationException("Invalid type " + str(type(temp)) + " for parameter " + self.requestmapper[param]['config'] \ + ". It is needed a " + str(mustbetype) + ".") elif self.requestmapper[param]['default'] is not None: configreq[param] = self.requestmapper[param]['default'] temp = self.requestmapper[param]['default'] elif self.requestmapper[param]['required']: raise ConfigurationException("Missing parameter " + self.requestmapper[param]['config'] + " from the configuration.") else: ## parameter not strictly required pass if param == "workflow": if mustbetype == type(self.requestname): configreq["workflow"] = self.requestname elif param in ['savelogsflag','publication','nonprodsw','ignorelocality','saveoutput']:#TODO use clientmappig to do this configreq[param] = 1 if temp else 0 elif param in ['dbsurl','publishdbsurl']: if param == 'dbsurl': dbstype = 'reader' elif param == 'publishdbsurl': dbstype = 'writer' alloweddbsurls = DBSURLS[dbstype].values() alloweddbsurlsaliases = DBSURLS[dbstype].keys() if configreq[param] in alloweddbsurlsaliases: configreq[param] = DBSURLS[dbstype][configreq[param]] else: if configreq[param].rstrip('/') in alloweddbsurls: configreq[param] = configreq[param].rstrip('/') else: raise ConfigurationException("Invalid argument " + configreq[param] + " for parameter " + self.requestmapper[param]['config'] + " in the configuration.") if (configreq['saveoutput'] or configreq['savelogsflag']) and 'asyncdest' not in configreq: raise ConfigurationException("Missing parameter " + self.requestmapper['asyncdest']['config'] + " from the configuration.") # Add debug parameters to the configreq dict configreq['oneEventMode'] = int(oneEventMode) jobconfig = {} self.configuration.JobType.proxyfilename = self.proxyfilename self.configuration.JobType.capath = HTTPRequests.getCACertPath() #get the backend URLs from the server external configuration serverBackendURLs = server_info('backendurls', self.serverurl, self.proxyfilename, self.getUrl(self.instance, resource='info')) #if cacheSSL is specified in the server external configuration we will use it to upload the sandbox (baseURL will be ignored) self.configuration.JobType.filecacheurl = serverBackendURLs['cacheSSL'] if 'cacheSSL' in serverBackendURLs else None pluginParams = [ self.configuration, self.logger, os.path.join(self.requestarea, 'inputs') ] if getattr(self.configuration.JobType, 'pluginName', None) is not None: jobtypes = getJobTypes() plugjobtype = jobtypes[upper(self.configuration.JobType.pluginName)](*pluginParams) inputfiles, jobconfig, isbchecksum = plugjobtype.run(configreq) else: fullname = self.configuration.JobType.externalPluginFile basename = os.path.basename(fullname).split('.')[0] plugin = addPlugin(fullname)[basename] pluginInst = plugin(*pluginParams) inputfiles, jobconfig, isbchecksum = pluginInst.run(configreq) if not configreq['publishname']: configreq['publishname'] = isbchecksum else: configreq['publishname'] = "%s-%s" %(configreq['publishname'], isbchecksum) configreq.update(jobconfig) server = HTTPRequests(self.serverurl, self.proxyfilename, self.proxyfilename, version=__version__) self.logger.info("Sending the request to the server") self.logger.debug("Submitting %s " % str( configreq ) ) dictresult, status, reason = server.put( self.uri, data = self._encodeRequest(configreq) ) self.logger.debug("Result: %s" % dictresult) if status != 200: msg = "Problem sending the request:\ninput:%s\noutput:%s\nreason:%s" % (str(configreq), str(dictresult), str(reason)) raise RESTCommunicationException(msg) elif dictresult.has_key("result"): uniquerequestname = dictresult["result"][0]["RequestName"] else: msg = "Problem during submission, no request ID returned:\ninput:%s\noutput:%s\nreason:%s" \ % (str(configreq), str(dictresult), str(reason)) raise RESTCommunicationException(msg) tmpsplit = self.serverurl.split(':') createCache(self.requestarea, tmpsplit[0], tmpsplit[1] if len(tmpsplit)>1 else '', uniquerequestname, voRole=self.voRole, voGroup=self.voGroup, instance=self.instance, originalConfig = self.configuration) self.logger.info("%sSuccess%s: Your task has been delivered to the CRAB3 server." %(colors.GREEN, colors.NORMAL)) if not self.options.wait: self.logger.info("Please use 'crab status' to check how the submission process proceed") self.logger.debug("Request ID: %s " % uniquerequestname) if self.options.wait: self.checkStatusLoop(server,uniquerequestname) return uniquerequestname
class FileTransfersTest(unittest.TestCase): """ _DashboardAPITest_ Unit tests for the FileTransfers API """ def setUp(self): """ Setup for unit tests """ self.server = HTTPRequests(os.environ['SERVER_HOST'], os.environ['X509_USER_PROXY'], os.environ['X509_USER_PROXY']) self.lfnBase = '/store/temp/user/%s/my_cool_dataset-%s/file-%s-%s.root' self.fileDoc = {'id': 'OVERWRITE', 'username': '******', 'taskname': 'OVERWRITE', 'start_time': 0, 'destination': 'T2_CH_CERN', 'destination_lfn': 'OVERWRITE', 'source': 'T2_US_Caltech', 'source_lfn': 'OVERWRITE', 'filesize': random.randint(1, 9999), 'publish': 1, 'transfer_state': 'OVERWRITE', 'publication_state': 'OVERWRITE', 'job_id': 1, 'job_retry_count': 0, 'type': 'log', 'rest_host': 'cmsweb.cern.ch', 'rest_uri': '/crabserver/prod/'} self.ids = [] self.users = ['jbalcas', 'mmascher', 'dciangot', 'riahi', 'erupeika', 'sbelforte'] # just random users for tests self.tasks = {} self.totalFiles = 10 def testFileTransferPUT(self): """ _testFileTransferPUT_ Just test simple testFileTransferPUT with fake data """ # We just sent fake data which is not monitored by dashboard. # Also only the first time to decide is publication ON or NOT for user in self.users: timestamp = time.strftime('%y%m%d_%H%M%S', time.gmtime()) for i in range(self.totalFiles): now = int(time.time()) # Generate a taskname workflowName = "" taskname = "" if user not in self.tasks: workflowName = "".join([random.choice(string.ascii_lowercase) for _ in range(20)]) + "_" + str(now) publicationState = random.choice(['NEW', 'NOT_REQUIRED']) else: workflowName = self.tasks[user]['workflowName'] publicationState = self.tasks[user]['publication'] transferState = random.choice(['NEW', 'DONE']) taskname = generateTaskName(user, workflowName, timestamp) finalLfn = self.lfnBase % (user, workflowName, i, random.randint(1, 9999)) idHash = getHashLfn(finalLfn) self.fileDoc['id'] = idHash self.fileDoc['job_id'] = i self.fileDoc['username'] = user self.fileDoc['taskname'] = taskname self.fileDoc['start_time'] = int(time.time()) self.fileDoc['source_lfn'] = finalLfn self.fileDoc['destination_lfn'] = finalLfn self.fileDoc['transfer_state'] = transferState self.fileDoc['publication_state'] = publicationState print(self.fileDoc) self.server.put('/crabserver/dev/fileusertransfers', data=encodeRequest(self.fileDoc)) # if I will put the same doc twice, it should raise an error. # self.server.put('/crabserver/dev/fileusertransfers', data=urllib.urlencode(self.fileDoc)) # This tasks are for the future and next calls if user not in self.tasks: self.tasks[user] = {'workflowName': workflowName, 'taskname': taskname, 'listOfIds': [], 'publication': publicationState, 'toTransfer': 0, 'toPublish': 0, 'total': self.totalFiles} if self.tasks[user]['publication'] == 'NEW': self.tasks[user]['toPublish'] += 1 if transferState == 'NEW': self.tasks[user]['toTransfer'] += 1 self.tasks[user]['listOfIds'].append(idHash) # This should raise an error for username in self.tasks: taskname = self.tasks[username]['taskname'] for query in ['getTransferStatus', 'getPublicationStatus']: result = self.server.get('/crabserver/dev/fileusertransfers', data=encodeRequest({'subresource': query, 'username': username, 'taskname': taskname})) print(result) print(result[0]['result']) taskInfoDict = oracleOutputMapping(result, 'id') print(taskInfoDict) for key, docDict in taskInfoDict.items(): result = self.server.get('/crabserver/dev/fileusertransfers', data=encodeRequest({'subresource': 'getById', 'id': key})) randomUsers = random.sample(set(self.users), 3) # Take half of the users and kill their transfers for specific task for username in randomUsers: taskname = self.tasks[username]['taskname'] result = self.server.post('/crabserver/dev/fileusertransfers', data=encodeRequest({'subresource': 'killTransfers', 'username': username, 'taskname': taskname})) print(result) # oneUser is left for killing a list of IDs # leftUsers will be killing transfers one by one for specific id. leftUsers = list(set(self.users) - set(randomUsers)) oneUser = random.sample(set(leftUsers), 1) leftUsers = list(set(leftUsers) - set(oneUser)) for username in leftUsers: # First get all left ids for this users result = self.server.get('/crabserver/dev/fileusertransfers', data=encodeRequest({'subresource': 'getTransferStatus', 'username': username, 'taskname': self.tasks[username]['taskname']})) resultOut = oracleOutputMapping(result, None) print("**"*50) for outDict in resultOut: print(outDict) result = self.server.post('/crabserver/dev/fileusertransfers', data=encodeRequest({'subresource': 'killTransfersById', 'username': username, 'listOfIds': outDict['id']})) print(result) print(resultOut) print(result) for username in oneUser: result = self.server.post('/crabserver/dev/fileusertransfers', data=encodeRequest({'subresource': 'killTransfersById', 'username': username, 'listOfIds': self.tasks[username]['listOfIds']}, ['listOfIds'])) # As it asks to kill all which are in new, need to double check what we submitted before and if the output of killed is correct print(result) print(self.tasks[username])
def __call__(self): valid = False configmsg = 'Default' if not os.path.isfile(self.options.config): raise MissingOptionException("Configuration file '%s' not found" % self.options.config) self.logger.debug("Started submission") ######### Check if the user provided unexpected parameters ######## #init the dictionary with all the known parameters SpellChecker.DICTIONARY = SpellChecker.train( [ val['config'] for _, val in self.requestmapper.iteritems() if val['config'] ] + \ [ x for x in self.otherConfigParams ] ) #iterate on the parameters provided by the user for section in self.configuration.listSections_(): for attr in getattr(self.configuration, section).listSections_(): par = (section + '.' + attr) #if the parameter is not know exit, but try to correct it before if not SpellChecker.is_correct( par ): msg = 'The parameter %s is not known.' % par msg += '' if SpellChecker.correct(par) == par else ' Did you mean %s?' % SpellChecker.correct(par) raise ConfigurationException(msg) #usertarball and cmsswconfig use this parameter and we should set it up in a correct way self.configuration.General.serverUrl = self.serverurl uniquerequestname = None self.logger.debug("Working on %s" % str(self.requestarea)) configreq = {} for param in self.requestmapper: mustbetype = getattr(types, self.requestmapper[param]['type']) if self.requestmapper[param]['config']: attrs = self.requestmapper[param]['config'].split('.') temp = self.configuration for attr in attrs: temp = getattr(temp, attr, None) if temp is None: break if temp: if mustbetype == type(temp): configreq[param] = temp else: raise ConfigurationException("Invalid type " + str(type(temp)) + " for parameter " + self.requestmapper[param]['config'] \ + ". It is needed a " + str(mustbetype) + ".") elif self.requestmapper[param]['default'] is not None: configreq[param] = self.requestmapper[param]['default'] elif self.requestmapper[param]['required']: raise ConfigurationException("Missing parameter " + self.requestmapper[param]['config'] + " from the configuration.") else: ## parameter not strictly required pass if param == "workflow": if mustbetype == type(self.requestname): configreq["workflow"] = self.requestname elif param == "savelogsflag": configreq["savelogsflag"] = 1 if temp else 0 elif param == "publication": configreq["publication"] = 1 if temp else 0 elif param == "blacklistT1": blacklistT1 = self.voRole != 't1access' #if the user choose to remove the automatic T1 blacklisting and has not the t1acces role if getattr (self.configuration.Site, 'removeT1Blacklisting', False) and blacklistT1: self.logger.info("WARNING: You disabled the T1 automatic blacklisting without having the t1access role") blacklistT1 = False configreq["blacklistT1"] = 1 if blacklistT1 else 0 jobconfig = {} self.configuration.JobType.proxyfilename = self.proxyfilename self.configuration.JobType.capath = HTTPRequests.getCACertPath() #get the backend URLs from the server external configuration serverBackendURLs = server_info('backendurls', self.serverurl, self.proxyfilename, self.getUrl(self.instance, resource='info')) #if cacheSSL is specified in the server external configuration we will use it to upload the sandbox (baseURL will be ignored) self.configuration.JobType.filecacheurl = serverBackendURLs['cacheSSL'] if 'cacheSSL' in serverBackendURLs else None #otherwise we will contact the baseurl to get the cache hostname self.configuration.JobType.baseurl = serverBackendURLs['baseURL'] pluginParams = [ self.configuration, self.logger, os.path.join(self.requestarea, 'inputs') ] if getattr(self.configuration.JobType, 'pluginName', None) is not None: jobtypes = getJobTypes() plugjobtype = jobtypes[upper(self.configuration.JobType.pluginName)](*pluginParams) inputfiles, jobconfig, isbchecksum = plugjobtype.run(configreq) else: fullname = self.configuration.JobType.externalPluginFile basename = os.path.basename(fullname).split('.')[0] plugin = addPlugin(fullname)[basename] pluginInst = plugin(*pluginParams) inputfiles, jobconfig, isbchecksum = pluginInst.run(configreq) if not configreq['publishname']: configreq['publishname'] = isbchecksum else: configreq['publishname'] = "%s-%s" %(configreq['publishname'], isbchecksum) configreq.update(jobconfig) server = HTTPRequests(self.serverurl, self.proxyfilename, self.proxyfilename, version=__version__) self.logger.info("Sending the request to the server") self.logger.debug("Submitting %s " % str( configreq ) ) dictresult, status, reason = server.put( self.uri, data = self._encodeRequest(configreq) ) self.logger.debug("Result: %s" % dictresult) if status != 200: msg = "Problem sending the request:\ninput:%s\noutput:%s\nreason:%s" % (str(configreq), str(dictresult), str(reason)) raise RESTCommunicationException(msg) elif dictresult.has_key("result"): uniquerequestname = dictresult["result"][0]["RequestName"] else: msg = "Problem during submission, no request ID returned:\ninput:%s\noutput:%s\nreason:%s" \ % (str(configreq), str(dictresult), str(reason)) raise RESTCommunicationException(msg) tmpsplit = self.serverurl.split(':') createCache(self.requestarea, tmpsplit[0], tmpsplit[1] if len(tmpsplit)>1 else '', uniquerequestname, voRole=self.voRole, voGroup=self.voGroup, instance=self.instance) self.logger.info("Submission completed") self.logger.debug("Request ID: %s " % uniquerequestname) self.logger.debug("Ended submission") return uniquerequestname