def setUp(self): self.returnObject = None self.toCheck = {} def execute(command, timeout=60, env=None, cwd=None, shell=False): import inspect frame = inspect.currentframe() fedInVars = inspect.getargvalues(frame).locals del frame for key, value in self.toCheck.iteritems(): if key in fedInVars: self.assertEqual(fedInVars[key], value) return self.returnObject def add_process(this, command, command_args=(), command_kwargs={}, timeout=60, env=None, cwd=None, shell=False, priority=5, callback_func=None, callback_args=(), callback_kwargs={}): import inspect frame = inspect.currentframe() fedInVars = inspect.getargvalues(frame).locals del frame for key, value in self.toCheck.iteritems(): if key in fedInVars: self.assertEqual(fedInVars[key], value) return self.returnObject self.db = DiracBase() from Ganga.Core.GangaThread.WorkerThreads.WorkerThreadPool import WorkerThreadPool setattr(sys.modules[self.db.__module__], 'execute', execute) setattr(WorkerThreadPool, "add_process", add_process) self.script = """
def db(): """Provides a DiracBase object per test function""" return DiracBase()
def db(): """Provides a DiracBase object per test function""" load_config_files() from GangaDirac.Lib.Backends.DiracBase import DiracBase yield DiracBase() clear_config()
class TestDiracBase(GangaGPITestCase): def setUp(self): self.returnObject = None self.toCheck = {} def execute(command, timeout=60, env=None, cwd=None, shell=False): import inspect frame = inspect.currentframe() fedInVars = inspect.getargvalues(frame).locals del frame for key, value in self.toCheck.iteritems(): if key in fedInVars: self.assertEqual(fedInVars[key], value) return self.returnObject def add_process(this, command, command_args=(), command_kwargs={}, timeout=60, env=None, cwd=None, shell=False, priority=5, callback_func=None, callback_args=(), callback_kwargs={}): import inspect frame = inspect.currentframe() fedInVars = inspect.getargvalues(frame).locals del frame for key, value in self.toCheck.iteritems(): if key in fedInVars: self.assertEqual(fedInVars[key], value) return self.returnObject self.db = DiracBase() from Ganga.Core.GangaThread.WorkerThreads.WorkerThreadPool import WorkerThreadPool setattr(sys.modules[self.db.__module__], 'execute', execute) setattr(WorkerThreadPool, "add_process", add_process) self.script = """ # dirac job created by ganga from DIRAC.Core.Base.Script import parseCommandLine parseCommandLine() ###DIRAC_IMPORT### ###DIRAC_JOB_IMPORT### dirac = ###DIRAC_OBJECT### j = ###JOB_OBJECT### # default commands added by ganga j.setName('###NAME###') j.setApplicationScript('###APP_NAME###','###APP_VERSION###','###APP_SCRIPT###',logFile='###APP_LOG_FILE###') j.setRootPythonScript('###ROOTPY_VERSION###', '###ROOTPY_SCRIPT###', ###ROOTPY_ARGS###, '###ROOTPY_LOG_FILE###') j.setRootMacro('###ROOT_VERSION###', '###ROOT_MACRO###', ###ROOT_ARGS###, '###ROOT_LOG_FILE###') j.setExecutable('###EXE###','###EXE_ARG_STR###','###EXE_LOG_FILE###') j.setExecutionEnv(###ENVIRONMENT###) j.setInputSandbox(##INPUT_SANDBOX##) j.setOutputSandbox(###OUTPUT_SANDBOX###) j.setInputData(###INPUTDATA###) j.setParametricInputData(###PARAMETRIC_INPUTDATA###) j.setOutputData(###OUTPUTDATA###,OutputPath='###OUTPUT_PATH###',OutputSE=###OUTPUT_SE###) j.setSystemConfig('###PLATFORM###') # <-- user settings ###SETTINGS### # user settings --> # diracOpts added by user ###DIRAC_OPTS### # submit the job to dirac result = dirac.submit(j) output(result) """ def test__setup_subjob_dataset(self): self.assertEqual(self.db._setup_subjob_dataset([]), None, 'Not None') def test__addition_sandbox_content(self): self.assertEqual( self.db._addition_sandbox_content(None), [], 'Not empty list') def test__setup_bulk_subjobs(self): from Ganga.Core import BackendError from Ganga.GPIDev.Lib.Dataset.Dataset import Dataset fd, name = tempfile.mkstemp() file = os.fdopen(fd, 'w') file.write(self.script.replace('###PARAMETRIC_INPUTDATA###', str([['a'], ['b']]))) file.close() self.assertRaises(BackendError, self.db._setup_bulk_subjobs, [], name) d = Dirac() j = Job(application=DaVinci(), splitter=SplitByFiles(), # merger=SmartMerger(), inputdata=Dataset(), backend=d) d._impl._parent = j._impl # self.db._parent = j._impl dirac_ids = [123, 456] def _setup_subjob_dataset(dataset): self.assertTrue( dataset in [['a'], ['b']], 'dataset not passed properly') return None setattr(self.db, '_setup_subjob_dataset', _setup_subjob_dataset) self.assertTrue( d._impl._setup_bulk_subjobs(dirac_ids, name), 'didnt run') self.assertEqual(len(j.subjobs), len(dirac_ids), 'didnt work') for id, backend_id, subjob in zip(range(len(dirac_ids)), dirac_ids, j.subjobs): self.assertEqual(id, subjob.id, 'ids dont match') self.assertEqual( backend_id, subjob.backend.id, 'backend.ids dont match') self.assertTrue(isinstance( subjob.application._impl, j.application._impl.__class__), 'apps dont match') self.assertEqual(subjob.splitter, None, 'splitter not done') # self.assertEqual(subjob.merger, None,'mergers dont match') # self.assertEqual(subjob.inputdata, None)#,'inputdata dont match') self.assertTrue(isinstance( subjob.backend._impl, j.backend._impl.__class__), 'backend dont match') os.remove(name) def test__common_submit(self): from Ganga.Core import BackendError j = Job(backend=self.db) self.db._parent = j._impl fd, name = tempfile.mkstemp() file = os.fdopen(fd, 'w') file.write(self.script.replace('###PARAMETRIC_INPUTDATA###', str([['a'], ['b']]))) file.close() self.returnObject = {} # class errorserver: # def execute(this, dirac_cmd): # return {} self.db.id = 1234 self.db.actualCE = 'test' self.db.status = 'test' self.assertRaises(BackendError, self.db._common_submit, name) # errorserver()) self.assertEqual(self.db.id, None, 'id not None') self.assertEqual(self.db.actualCE, None, 'actualCE not None') self.assertEqual(self.db.status, None, 'status not None') # class server: # def __init__(this, list_return=False): # this.list_return = list_return # def execute(this, dirac_cmd): # self.assertEqual(dirac_cmd,"execfile('%s')"%name,'cmd wrong') # if this.list_return is True: # return {'OK':True, 'Value': [123,456]} # return {'OK':True, 'Value': 12345} self.toCheck = {'command': "execfile('%s')" % name} self.returnObject = {'OK': True, 'Value': 12345} self.assertTrue(self.db._common_submit(name)) self.assertEqual(self.db.id, 12345, 'id not set') def _setup_bulk_subjobs(dirac_ids, dirac_script): self.assertEqual(dirac_ids, [123, 456], 'ids not equal') self.assertEqual(dirac_script, name, 'dirac script not equal') return True setattr(self.db, '_setup_bulk_subjobs', _setup_bulk_subjobs) self.returnObject = {'OK': True, 'Value': [123, 456]} self.assertTrue(self.db._common_submit(name)) os.remove(name) def test_submit(self): j = Job(backend=self.db) self.db._parent = j._impl file1 = tempfile.NamedTemporaryFile('w') file2 = tempfile.NamedTemporaryFile('w') file3 = tempfile.NamedTemporaryFile('w') sjc = StandardJobConfig(exe=self.script, inputbox=[File(file1.name)._impl, File(file2.name)._impl, File(file3.name)._impl], outputbox=['d', 'e', 'f']) def _addition_sandbox_content(subjobconfig): self.assertEqual(subjobconfig, sjc, 'config objects not equal') return ['g'] def _common_submit(dirac_script): # this needs to change to workerpool #from GangaDirac.Lib.Server.DiracClient import DiracClient #self.assertTrue(isinstance(server, DiracClient),'not a dirac client') f = open(dirac_script, 'r') script = f.read() f.close() self.assertNotEqual(script, self.script, 'script not changed') self.assertEqual(self.script.replace('##INPUT_SANDBOX##', str(['a', 'b', 'c'] + [os.path.join(j._impl.getInputWorkspace().getPath(), '_input_sandbox_0.tgz')] + ['g'])), script, 'script not what it should be') return True setattr( self.db, '_addition_sandbox_content', _addition_sandbox_content) setattr(self.db, '_common_submit', _common_submit) self.assertTrue(self.db.submit(sjc, ['a', 'b', 'c']), 'didnt run') file1.close() file2.close() file3.close() def test_resubmit(self): def _resubmit(): return '_resubmit run ok' setattr(self.db, '_resubmit', _resubmit) self.assertEqual(self.db.resubmit(), '_resubmit run ok') def test__resubmit(self): from Ganga.Core import BackendError # class server: # def execute(this, dirac_cmd): # return {} def _common_submit(dirac_script): return '_comon_submit run ok' setattr(self.db, '_common_submit', _common_submit) masterj = Job(backend=self.db) j = Job(backend=self.db) j._impl.master = masterj._impl # problem as keeps finding job 0 in main repository which has this # 'dirac-script' file self.db._parent = masterj._impl self.returnObject = {} self.assertRaises(BackendError, self.db._resubmit) # server()) ## self.db._parent = j._impl # self.assertRaises(BackendError, # self.db._resubmit, # server()) # Come back to this def test_reset(self): j = Job(backend=self.db)._impl self.db._parent = j self.db.getJobObject().subjobs = [ Job(backend=self.db)._impl, Job(backend=self.db)._impl] for j in self.db.getJobObject().subjobs: j.status = 'completing' disallowed_status = ['submitting', 'killed'] for status in disallowed_status: self.db.getJobObject().status = status self.db.reset() self.assertEqual( self.db.getJobObject().status, status, 'status shouldn\'t have changed') self.db.getJobObject().status = 'completing' self.db.reset() self.assertEqual( self.db.getJobObject().status, 'submitted', 'didn\t reset job') self.assertNotEqual([j.status for j in self.db.getJobObject().subjobs], [ 'submitted', 'submitted'], 'subjobs not reset properly') self.db.reset(doSubjobs=True) self.assertEqual([j.status for j in self.db.getJobObject().subjobs], [ 'submitted', 'submitted'], 'subjobs not reset properly') for j in self.db.getJobObject().subjobs: j.status = 'completed' self.db.reset(doSubjobs=True) self.assertNotEqual([j.status for j in self.db.getJobObject().subjobs], [ 'submitted', 'submitted'], 'subjobs not supposed to reset') def test_kill(self): # class errorserver: # def execute(this, dirac_cmd): # return {} # setattr(DiracBase,'dirac_ganga_server',errorserver()) self.returnObject = {} self.db.id = 1234 from Ganga.Core import BackendError self.assertRaises(BackendError, self.db.kill) # class server: # def execute(this, dirac_cmd): # self.assertEqual(dirac_cmd,'kill(1234)','command not right') # return {'OK': True} # setattr(DiracBase,'dirac_ganga_server',server()) self.toCheck = {'command': 'kill(1234)'} self.returnObject = {'OK': True} self.assertTrue(self.db.kill(), 'didn\'t run properly') def test_peek(self): # class server: # def execute(this, dirac_cmd): # self.assertEqual(dirac_cmd,'peek(1234)') # return {'OK':True,'Value':True} # setattr(DiracBase,'dirac_ganga_server',server()) self.db.id = 1234 self.toCheck = {'command': 'peek(1234)'} self.returnObject = {'OK': True, 'Value': True} self.db.peek() def test_getOutputSandbox(self): j = Job(backend=self.db) self.db._parent = j._impl self.db.id = 1234 tempDir = j._impl.getOutputWorkspace().getPath() # class server: # def execute(this, dirac_cmd): # self.assertEqual(dirac_cmd, # "getOutputSandbox(1234,'%s')"%dir, # 'command not right') # return {'OK':True} # setattr(DiracBase,'dirac_ganga_server',server()) self.toCheck = {'command': "getOutputSandbox(1234,'%s')" % tempDir} self.returnObject = {'OK': True} self.assertTrue(self.db.getOutputSandbox(), 'didn\'t run') testDir = 'test_dir' self.toCheck = {'command': "getOutputSandbox(1234,'%s')" % testDir} self.assertTrue(self.db.getOutputSandbox(testDir), 'didn\'t run with modified dir') # class errorserver: # def execute(this, dirac_cmd): # return {} # setattr(DiracBase,'dirac_ganga_server',errorserver()) self.toCheck = {} self.returnObject = {} self.assertFalse(self.db.getOutputSandbox(testDir)), 'didn\'t fail gracefully' def test_removeOutputData(self): j = Job(backend=self.db) self.db._parent = j._impl ####################### class testfile: def __init__(this): pass def remove(this): return 27 ####################### import GangaDirac.Lib.Backends.DiracBase def outputfiles_foreach(job, file_type, func): import types self.assertTrue(isinstance(job, Job._impl)) if subjob: self.assertNotEqual(job.master, None) else: self.assertEqual(job.master, None) self.assertEqual(file_type, DiracFile._impl) self.assertEqual(type(func), types.FunctionType) self.assertEqual( func(testfile()), 27, 'Didn\'t call remove function') setattr(GangaDirac.Lib.Backends.DiracBase, 'outputfiles_foreach', outputfiles_foreach) subjob = False self.assertEqual(self.db.removeOutputData(), None) j._impl.subjobs = [Job()._impl, Job()._impl, Job()._impl] for sj in j._impl.subjobs: sj.master = j._impl subjob = True self.assertEqual(self.db.removeOutputData(), None) def test_getOutputData(self): j = Job(backend=self.db) self.db._parent = j._impl self.assertRaises(GangaException, self.db.getOutputData, '/false/dir') ####################### class testfile: def __init__(this, lfn, namePattern): this.lfn = lfn this.namePattern = namePattern def get(this): this.check = 42 test_files = [testfile('a', 'alpha'), testfile('', 'delta'), testfile('b', 'beta'), testfile('', 'bravo'), testfile('c', 'charlie'), testfile('', 'foxtrot')] ####################### import GangaDirac.Lib.Backends.DiracBase def outputfiles_iterator(job, file_type): import types self.assertTrue(isinstance(job, Job._impl)) if subjob: self.assertNotEqual(job.master, None) else: self.assertEqual(job.master, None) self.assertEqual(file_type, DiracFile._impl) return test_files setattr(GangaDirac.Lib.Backends.DiracBase, 'outputfiles_iterator', outputfiles_iterator) # master jobs ####################### subjob = False self.assertEqual(self.db.getOutputData(), ['a', 'b', 'c']) for f in test_files: if f.lfn in ['a', 'b', 'c']: self.assertEqual( f.localDir, j._impl.getOutputWorkspace().getPath()) self.assertEqual(f.check, 42, "didn't call get") #delattr(f, 'localDir') else: self.assertFalse(hasattr(f, 'localDir')) self.assertFalse(hasattr(f, 'check')) self.assertEqual( self.db.getOutputData(None, ['alpha', 'charlie']), ['a', 'c']) self.assertEqual(self.db.getOutputData( os.path.expanduser('~/gangadir_testing'), ['alpha', 'charlie']), ['a', 'c']) # subjobs ######################## j._impl.subjobs = [Job()._impl, Job()._impl, Job()._impl] i = 0 for sj in j._impl.subjobs: sj.master = j._impl sj.id = i i += 1 subjob = True self.assertEqual(self.db.getOutputData(), ['a', 'b', 'c'] * 3) self.assertEqual(self.db.getOutputData(None, ['beta']), ['b'] * 3) self.assertEqual(self.db.getOutputData( os.path.expanduser('~/gangadir_testing'), ['alpha', 'charlie']), ['a', 'c'] * 3) for i in range(3): self.assertTrue(os.path.isdir( os.path.join(os.path.expanduser('~/gangadir_testing'), '0.%d' % i))) os.rmdir( os.path.join(os.path.expanduser('~/gangadir_testing'), '0.%d' % i)) def test_getOutputDataLFNs(self): j = Job(backend=self.db) self.db._parent = j._impl ####################### class testfile: def __init__(this, lfn): this.lfn = lfn ####################### import GangaDirac.Lib.Backends.DiracBase def outputfiles_iterator(job, file_type): import types self.assertTrue(isinstance(job, Job._impl)) if subjob: self.assertNotEqual(job.master, None) else: self.assertEqual(job.master, None) self.assertEqual(file_type, DiracFile._impl) return [testfile('a'), testfile(''), testfile('b'), testfile(''), testfile('c'), testfile('')] setattr(GangaDirac.Lib.Backends.DiracBase, 'outputfiles_iterator', outputfiles_iterator) subjob = False self.assertEqual(self.db.getOutputDataLFNs(), ['a', 'b', 'c']) j._impl.subjobs = [Job()._impl, Job()._impl, Job()._impl] for sj in j._impl.subjobs: sj.master = j._impl subjob = True self.assertEqual(self.db.getOutputDataLFNs(), ['a', 'b', 'c'] * 3)