def runLocal(self, diracLHCb=None): """ The DiracLHCb (API) object is for local submission. A BKKClient might be needed. First, adds Ancestors (in any) to the InputData. """ if diracLHCb is not None: diracLHCb = diracLHCb else: diracLHCb = DiracLHCb() return diracLHCb.submitJob(self, mode='local')
def __init__(self): """constructor""" cmd.Cmd.__init__(self) self.prompt = "$[/]$" self.bk = LHCB_BKKDBClient() self.diracAPI = DiracLHCb() self.currentPath = '/' self.do_setDataQualityFlags('OK') self.saveParser = argparse.ArgumentParser(description="Save LFNS", prog='save') self.saveParser.add_argument('filename', type=str, help='file name') self.saveParser.add_argument("-f", "--format", help="txt or py") self.saveParser.add_argument("-n", "--num", help="number of files to be saved") self.saveParser.add_argument("-c", "--with-fileCatalog", help="save POOL XML catalog in a given site") self.sites = {} self.sites = DMSHelpers().getShortSiteNames(withStorage=False, tier=(0, 1))
def setUp(self): super(UserJobTestCase, self).setUp() print "\n \n ********************************* \n Running a new test \n *********************************" self.dLHCb = DiracLHCb() try: self.exeScriptLocation = find_all('exe-script.py', os.environ['WORKSPACE'], '/tests/Workflow/Integration')[0] except (IndexError, KeyError): self.exeScriptLocation = find_all('exe-script.py', rootPath, '/tests/Workflow/Integration')[0] try: self.exeScriptFromDIRACLocation = find_all( 'exe-script-fromDIRAC.py', os.environ['WORKSPACE'], '/tests/Workflow/Integration')[0] except (IndexError, KeyError): self.exeScriptFromDIRACLocation = find_all( 'exe-script-fromDIRAC.py', rootPath, '/tests/Workflow/Integration')[0] self.lhcbJobTemplate = LHCbJob() self.lhcbJobTemplate.setLogLevel('DEBUG') try: # This is the standard location in Jenkins self.lhcbJobTemplate.setInputSandbox( find_all('pilot.cfg', os.environ['WORKSPACE'] + '/PilotInstallDIR')[0]) except (IndexError, KeyError): self.lhcbJobTemplate.setInputSandbox( find_all('pilot.cfg', rootPath)[0]) self.lhcbJobTemplate.setConfigArgs('pilot.cfg')
def setUp(self): self.dirac = DiracLHCb() result = getProxyInfo() if result['Value']['group'] not in ['lhcb_user', 'dirac_user']: print "GET A USER GROUP" exit(1)
def test_Integration_User_Failing(self): """ This job will fail everything that can fail """ print "Submitting gaudiRun job (Gauss only) that will use a configuration file that contains wrong info" print "This will generate a local job" os.environ['JOBID'] = '12345' gaudirunJob = createJob() result = DiracLHCb().submitJob(gaudirunJob, mode='Local') self.assertFalse(result['OK']) del os.environ['JOBID']
def setUp(self): super(RegressionTestCase, self).setUp() self.diracLHCb = DiracLHCb() self.bkkClient = BookkeepingClient() exeScriptLoc = find_all('exe-script.py', '..', '/LHCbDIRAC/tests/Workflow/Regression')[0] helloWorldLoc = find_all('helloWorld.py', '..', '/LHCbDIRAC/tests/Workflow/Regression')[0] shutil.copyfile(exeScriptLoc, './exe-script.py') shutil.copyfile(helloWorldLoc, './helloWorld.py') helloWorldXMLLocation = find_all( 'helloWorld.xml', '..', '/LHCbDIRAC/tests/Workflow/Regression')[0] self.j_u_hello = LHCbJob(helloWorldXMLLocation) self.j_u_hello.setConfigArgs('pilot.cfg') helloWorldXMLFewMoreLocation = find_all( 'helloWorld.xml', '..', '/LHCbDIRAC/tests/Workflow/Regression')[0] self.j_u_helloPlus = LHCbJob(helloWorldXMLFewMoreLocation) self.j_u_helloPlus.setConfigArgs('pilot.cfg')
def setUp(self): self.dirac = DiracLHCb() result = getProxyInfo() if result['Value']['group'] not in ['lhcb_admin']: print "GET A ADMIN GROUP" exit(1) result = ResourceStatus().getElementStatus('PIC-USER', 'StorageElement', 'WriteAccess') if result['Value']['PIC-USER']['WriteAccess'].lower() != 'banned': print "BAN PIC-USER in writing! and then restart this test" exit(1) res = DataManager().getReplicas([ '/lhcb/user/f/fstagni/test/testInputFileSingleLocation.txt', '/lhcb/user/f/fstagni/test/testInputFile.txt' ]) if not res['OK']: print "DATAMANAGER.getRepicas failure: %s" % res['Message'] exit(1) if res['Value']['Failed']: print "DATAMANAGER.getRepicas failed for something: %s" % res[ 'Value']['Failed'] exit(1) replicas = res['Value']['Successful'] if replicas['/lhcb/user/f/fstagni/test/testInputFile.txt'].keys() != [ 'CERN-USER', 'IN2P3-USER' ]: print "/lhcb/user/f/fstagni/test/testInputFile.txt locations are not correct" if replicas[ '/lhcb/user/f/fstagni/test/testInputFileSingleLocation.txt'].keys( ) != ['CERN-USER']: print "/lhcb/user/f/fstagni/test/testInputFileSingleLocation.txt locations are not correct"
# dirac job created by ganga from DIRAC.DataManagementSystem.Utilities.DMSHelpers import DMSHelpers from LHCbDIRAC.Interfaces.API.LHCbJob import LHCbJob from LHCbDIRAC.Interfaces.API.DiracLHCb import DiracLHCb j = LHCbJob() dirac = DiracLHCb() # default commands added by ganga j.setName("helloWorld-test") j.setInputSandbox([ '/afs/cern.ch/user/f/fstagni/userJobs/_inputHello.tgz', '/afs/cern.ch/user/f/fstagni/userJobs/hello-script.py' ]) j.setExecutable("exe-script.py", "", "Ganga_Executable.log") # <-- user settings j.setCPUTime(172800) try: tier1s = DMSHelpers().getTiers(tier=(0, 1)) except AttributeError: tier1s = [ 'LCG.CERN.cern', 'LCG.CNAF.it', 'LCG.GRIDKA.de', 'LCG.IN2P3.fr', 'LCG.NIKHEF.nl', 'LCG.PIC.es', 'LCG.RAL.uk', 'LCG.RRCKI.ru', 'LCG.SARA.nl' ] j.setBannedSites(tier1s) # user settings --> # print j.workflow
import os import sys import time import datetime import glob import pickle import inspect ## NB parseCommandLine first then import Dirac!! from DIRAC.Core.Base.Script import parseCommandLine parseCommandLine() from LHCbDIRAC.Interfaces.API.DiracLHCb import DiracLHCb from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin diraclhcb = DiracLHCb() # Write to output pipe #/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/ #def output(object): # print(pickle.dumps(object), file=sys.stdout) # DiracLHCb commands #/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/ #def getRootVersions(): output( diraclhcb.getRootVersions() ) #def getSoftwareVersions(): output( diraclhcb.getSoftwareVersions() ) def bkQueryDict(dict): output(diraclhcb.bkQuery(dict))
jobs[job][jobID]['Retrieved'] = '0' jobs[job][jobID]['Sandbox'] = '' jobs[job][jobID]['Time'] = time.ctime().replace(' ', '_') flags[id] = 0 #DIRAC objects instantiation from DIRAC.Core.Base.Script import parseCommandLine parseCommandLine() from LHCbDIRAC.Interfaces.API.DiracLHCb import DiracLHCb from LHCbDIRAC.Interfaces.API.LHCbJob import LHCbJob from DIRAC.Interfaces.API.JobRepository import JobRepository dirac = DiracLHCb() # READING REPOSITORY repo = shelve.open(jobsdir + 'repo_DK.db', 'c', 2) jobList = [] downloadStates = ['Done', 'Failed', 'Completed'] for j, jdict in repo.iteritems(): if jdict['State'] in downloadStates and int(jdict['Retrieved']) == 0: jobList.append({j: jdict}) #SPLITTING JOBLIST BY THREAD nThread = int(10) if nThread > len(jobList): nThread = len(jobList) njobsPerList = len( jobList) / nThread #Number of jobs to submit for each thread jobListPerThread = [
def setUp(self): super(RegressionTestCase, self).setUp() self.diracLHCb = DiracLHCb()
from DIRAC.Core.Base.Script import parseCommandLine parseCommandLine() from DIRAC import gLogger, rootPath from DIRAC.tests.Utilities.utils import find_all from LHCbDIRAC.Interfaces.API.LHCbJob import LHCbJob from LHCbDIRAC.Interfaces.API.DiracLHCb import DiracLHCb #from tests.Workflow.Integration.Test_UserJobs import createJob gLogger.setLevel('DEBUG') cwd = os.path.realpath('.') dirac = DiracLHCb() # Simple Hello Word job to DIRAC.Jenkins.ch gLogger.info("\n Submitting hello world job targeting DIRAC.Jenkins.ch") helloJ = LHCbJob() helloJ.setName("helloWorld-TEST-TO-Jenkins") try: helloJ.setInputSandbox([find_all('exe-script.py', rootPath, '/DIRAC/tests/Workflow/')[0]]) except IndexError: helloJ.setInputSandbox([find_all('exe-script.py', os.environ['WORKSPACE'], '/DIRAC/tests/Workflow/')[0]]) helloJ.setExecutable("exe-script.py", "", "helloWorld.log") helloJ.setCPUTime(17800) helloJ.setDestination('DIRAC.Jenkins.ch') result = dirac.submitJob(helloJ) gLogger.info("Hello world job: ", result) if not result['OK']:
#!/storage/gpfs_lhcb_t3_01/lhcb_soft/lcg/external/Python/2.6.5p2/x86_64-slc5-gcc43-opt/bin/python import shelve, time #DIRAC objects instantiation from DIRAC.Core.Base.Script import parseCommandLine parseCommandLine() from LHCbDIRAC.Interfaces.API.DiracLHCb import DiracLHCb from LHCbDIRAC.Interfaces.API.LHCbJob import LHCbJob from DIRAC.Interfaces.API.JobRepository import JobRepository dirac = DiracLHCb() jobs = shelve.open('outGrid/repo_DK.db', 'c', 2) result = dirac.status(jobs.keys()) finalReport = {} if result['OK']: newjobs = result['Value'] for jID, jDict in jobs.iteritems(): job = jobs[jID] if not newjobs.has_key(int(jID)): continue newjob = newjobs[int(jID)] if newjob.has_key('Status'): job['State'] = newjob['Status'] if newjob.has_key('Site'): job['Site'] = newjob['Site'] job['Time'] = time.ctime().replace(' ', '_') if not finalReport.has_key(job['State']): finalReport[job['State']] = 1 else: finalReport[job['State']] += 1 jobs[jID] = job jobs.close()
class LHCbBookkeepingCLI(cmd.Cmd): """class""" ############################################################################# def __init__(self): """constructor""" cmd.Cmd.__init__(self) self.prompt = "$[/]$" self.bk = LHCB_BKKDBClient() self.diracAPI = DiracLHCb() self.currentPath = '/' self.do_setDataQualityFlags('OK') self.saveParser = argparse.ArgumentParser(description="Save LFNS", prog='save') self.saveParser.add_argument('filename', type=str, help='file name') self.saveParser.add_argument("-f", "--format", help="txt or py") self.saveParser.add_argument("-n", "--num", help="number of files to be saved") self.saveParser.add_argument("-c", "--with-fileCatalog", help="save POOL XML catalog in a given site") self.sites = {} self.sites = DMSHelpers().getShortSiteNames(withStorage=False, tier=(0, 1)) ############################################################################# def addCurrentPath(self, path): """add a path""" if path[0] != '/' and len(self.currentPath) == 1: self.currentPath += path else: self.currentPath += '/' + path ############################################################################# def __printPrompt(self, path=None): """prints the prompt""" if path is None: path = '/' self.prompt = '$[' + path + ']$' ############################################################################# def __bklist(self, path): """list a path""" retVal = self.bk.list(path) return retVal def __bkListAll(self, path): """list the directory with metadata. This is equivalent to ls -a """ if path == '': path = self.currentPath res = self.bk.list(path) for i in res: print(i['name']) for j in i: if j not in ['fullpath', 'selection', 'expandable', 'method', 'level', 'name']: print(' ', j, i[j]) ############################################################################# def __checkDirectory(self, path): """is empty directory""" res = self.bk.list(path) retValue = False if res: retValue = True return retValue ############################################################################# def __rootDirectory(self): """root""" self.currentPath = '/' self.__printPrompt(self.currentPath) ############################################################################# def __oneLevelback(self, logical=False): """ cd .. """ path = self.currentPath.split('/') if path[0] == '' and path[1] == '': if not logical: self.currentPath = '/' self.__printPrompt(self.currentPath) else: return '/' else: newpath = '' for i in xrange(len(path) - 1): if path[i] != '': newpath += '/' + path[i] if newpath == '': newpath = '/' if not logical: self.currentPath = newpath self.__printPrompt(self.currentPath) else: return newpath ############################################################################# @staticmethod def help_ls(): """provides help""" print("Usage: ls [OPTION]... [FILE]...") print("List information about the FILEs (the current directory by default).") print(" Available options: -a") print('Usage: ls or ls -a') print('ls -a [FILE]') print('Note: You can do paging using the | more') print('For example: ls | more') @staticmethod def help_cd(): """help cd command""" print(" cd <dir>") print("cd ..") print("cd /") print("cd") ############################################################################# def do_ls(self, path): """ls command""" paging = False if path.find('|') > -1: tmpPath = path.split('|') path = '' for i in xrange(len(tmpPath) - 1): path += tmpPath[i].strip() paging = True text = '' if len(path) > 0 and path[0] == '-': try: if path[1] != 'a': print("ls: invalid option -- {}".format(path[1])) print("Try `help ls' for more information.") else: path = path[2:] self.__bkListAll(path) except IndexError as e: print("Invalid oprion:", e) elif path == '': res = self.__bklist(self.currentPath) for i in sorted(res): if paging: text += i['name'] + '\n' else: print(i['name']) else: res = self.__bklist(path) for i in sorted(res): if paging: text += i['name'] + '\n' else: print(i['name']) if paging: pydoc.ttypager(text) ############################################################################# def do_list(self, path): """list commamd""" pass ############################################################################# def do_save(self, command): """save command""" try: args = self.saveParser.parse_args(shlex.split(command)) except argparse.ArgumentError as exc: print(exc) except SystemExit: self.saveParser.print_help() return if args.format is None: args.format = 'txt' retVal = self.bk.list(self.currentPath) files = None if retVal: if 'FileName' not in retVal[0]: print('No file found belong to {} bookkeeping path!'.format(self.currentPath)) return else: files = retVal else: print(retVal) return if args.num is not None: try: nb = long(args.num) except ValueError as v: print(v) return nbOfFiles = nb else: nbOfFiles = len(files) lfns = {} for i in xrange(nbOfFiles): lfns[files[i]['FileName']] = files[i] if args.with_fileCatalog: site = args.with_fileCatalog lfnList = lfns.keys() totalFiles = len(lfnList) ff = args.filename.split('.') catalog = ff[0] + '.xml' retVal = self.diracAPI.getInputDataCatalog(lfnList, site, catalog, True) nbofsuccsessful = 0 if retVal['OK']: slist = {} faild = {} if 'Successful' in retVal['Value']: slist = retVal['Value']['Successful'] if 'Failed' in retVal['Value']: faild = retVal['Value']['Failed'] nbofsuccsessful = len(slist) nboffaild = len(faild) exist = {} for i in slist: exist[i] = lfns[i] self.bk.writeJobOptions(exist, args.filename, savedType=None, catalog=catalog, savePfn=slist) message = 'Total files:' + str(totalFiles) + '\n' if site is not None: if nbofsuccsessful: message += str(nbofsuccsessful) + ' found ' + site + '\n' if nboffaild: message += str(nboffaild) + ' not found ' + site print(message) return if args.format == 'txt': text = self.bk.writeJobOptions(lfns, '', args.format) elif args.format == 'py': text = self.bk.writeJobOptions(lfns, args.filename, args.format) with open(args.filename, 'w') as f: f.write(text) ############################################################################# def do_cd(self, path): """cd command""" newpath = self.currentPath + '/' + path if path == '': self.currentPath = '/' self.__printPrompt(self.currentPath) elif path == '..': self.__oneLevelback() elif path == '/': self.__rootDirectory() elif path[0] == '/': if self.__checkDirectory(path): self.currentPath = path self.__printPrompt(self.currentPath) else: print('No such file or directory.') elif self.__checkDirectory(newpath): self.addCurrentPath(path) self.__printPrompt(self.currentPath) else: print('No such file or directory.') ############################################################################# def do_pwd(self, path): """pwd command""" print(self.currentPath) ############################################################################# def do_queries(self, command=''): """execute query""" retVal = self.bk.getPossibleParameters() print('The following bookkeeping query types are available:') for i in retVal: print(' '.rjust(10) + i) print("You can change the query types using the 'use' command") ############################################################################# def do_use(self, command): """use command""" self.bk.setParameter(str(command)) self.do_cd('/') ############################################################################# @staticmethod def help_use(): """help of use command""" print('Usage:') print(' use type'.rjust(10)) print('Arguments:') print(' type: bookkeeping query type'.rjust(10)) print("The 'type' can be found using the 'queries' command!") print("EXAMPE:") print(' '.rjust(10) + "use 'Event type'") ############################################################################# @staticmethod def help_queries(): """help of queries command""" print("This method shows the available query types!") print("Usage:") print(" queries") print("You can choose a query type using the 'use' command ") print(" NOTE: the default query type is 'Configuration'") ############################################################################# def do_advanceQuery(self, command=''): """advancedQuery command""" self.bk.setAdvancedQueries(True) self.do_cd('/') ############################################################################# @staticmethod def help_advanceQuery(): """help""" print("It allows to see more level of the Bookkeeping Tree") print("Usage:") print(" advanceQuery") ############################################################################# def do_standardQuery(self, command=''): """command""" self.bk.setAdvancedQueries(False) self.do_cd('/') ############################################################################# @staticmethod def help_standardQuery(): """help""" print("This is used by default") print("It shows a reduced bookkeeping path.") print("Usage:") print(" standardQuery") ############################################################################# def do_dataQuality(self, command=''): """command""" print('The following Data Quality flags are available in the bookkeeping!') retVal = self.bk.getAvailableDataQuality() if retVal['OK']: for i in retVal['Value']: print(' '.ljust(10) + i) else: print(retVal["Message"]) print("To set the data quality flags you have to use 'setDataQualityFlags' command!") print("More information: 'help setDataQualityFlags'") ############################################################################# @staticmethod def help_dataQuality(): """help""" print('This command shows the available data quality flags.') print("Usage:") print(" dataQuality") print('To change the data quality flag use the setDataQualityFlags command') ############################################################################# def do_setDataQualityFlags(self, command): """command""" qualities = command.split(' ') if len(qualities) > 0: dataquality = {} for i in qualities: dataquality[i] = True self.bk.setDataQualities(dataquality) else: print('ERROR: Please give a data quality flag!') ############################################################################# def __moreInfoProcpass(self, command): """more information of a directory""" found = False retVal = self.bk.getProcessingPassSteps({'StepName': command}) if retVal['OK']: proc = retVal['Value'] print('{0} {1} step found in the bkk'.format(proc['TotalRecords'], command)) for i in proc['Records']: print(' '.ljust(5) + i) for j in proc['Records'][i]: print(' '.ljust(10) + str(j[0]) + ':' + str(j[1])) found = True else: print('ERROR: ', retVal['Message']) return found ############################################################################# def do_moreinfo(self, command=''): """more info command""" if command == '': previouspath = self.__oneLevelback(self.currentPath) values = self.__bklist(previouspath) name = self.currentPath.split('/') for i in values: if 'level' in i and i['level'] == 'FileTypes': path = self.currentPath retVal = self.bk.getLimitedFiles({'fullpath': str(path)}, ['nb'], -1, -1) print('The selected dataset is:') for i in retVal['Extras']['Selection']: print(''.ljust(5) + i + ' ' + str(retVal['Extras']['Selection'][i])) print('Statistics:') print(' '.ljust(5) + 'Number of files:' + str(retVal['TotalRecords'])) for i in retVal['Extras']['GlobalStatistics']: print(''.ljust(5) + i + ' ' + str(retVal['Extras']['GlobalStatistics'][i])) break if i['name'] == name[len(name) - 1]: for j in i: if j not in ['fullpath', 'selection', 'expandable', 'method', 'level', 'name']: print(' ', j, i[j]) if 'level' in i and i['level'] == 'Processing Pass': found = self.__moreInfoProcpass(command) break else: values = self.__bklist(self.currentPath) found = False for i in values: if i['name'] == command: for j in i: if j not in ['fullpath', 'selection', 'expandable', 'method', 'level', 'name']: print(' ', j, i[j]) found = True if 'level' in i and i['level'] == 'Processing Pass': found = self.__moreInfoProcpass(command) break if not found: print(" The '%s' does not found" % (command)) def do_sites(self, command): print("T0/1 sites: %s" % ','.join(site for site in self.sites)) @staticmethod def help_sites(self): """ help """ print("it returns a list of T1 sites") ############################################################################# @staticmethod def help_setDataQualityFlags(): """help""" print('This command allows to use different data quality flags.') print("Usage:") print(" setDataQualityFlags flag1 [flag2 flag3, ... flagN]") print("Arguments:") print(" flag[1...N]: Data quality flags.") print('For example:') print(' '.ljust(10) + 'setDataQualityFlags OK UNCHECKED') ############################################################################# @staticmethod def help_EOF(): """quit""" print("Quits the program") ############################################################################# def do_EOF(self, line): """quit command""" sys.exit() ############################################################################# def help_save(self): """help""" self.saveParser.print_help() ############################################################################# @staticmethod def help_moreinfo(): """help method""" print("Display the statistics of the selected data.") print("Usage:") print(" moreinfo")
def setUp( self ): cleanTestDir() self.dirac = DiracLHCb() gLogger.setLevel( 'DEBUG' ) print "\n\n****************************************************** Start running test"
from DIRAC.DataManagementSystem.Utilities.DMSHelpers import DMSHelpers try: from LHCbDIRAC.tests.Workflow.Integration.Test_UserJobs import createJob except ImportError: from tests.Workflow.Integration.Test_UserJobs import createJob gLogger.setLevel('DEBUG') cwd = os.path.realpath('.') ######################################################################################## gLogger.info("\n Submitting hello world job banning T1s") helloJ = LHCbJob() dirac = DiracLHCb() helloJ.setName("helloWorld-test-T2s") helloJ.setInputSandbox([ find_all('exe-script.py', rootPath, '/tests/System/GridTestSubmission')[0] ]) helloJ.setExecutable("exe-script.py", "", "helloWorld.log") helloJ.setCPUTime(17800) try: tier1s = DMSHelpers().getTiers(tier=(0, 1)) except AttributeError: tier1s = [ 'LCG.CERN.cern', 'LCG.CNAF.it', 'LCG.GRIDKA.de', 'LCG.IN2P3.fr', 'LCG.NIKHEF.nl', 'LCG.PIC.es', 'LCG.RAL.uk', 'LCG.RRCKI.ru',