def setUp(self): # initial setup self.inputsandbox = [File('a'), File('b'), File('c')] self.outputsandbox = ['d', 'e', 'f'] self.appconfig_inputbox = [ File('g')._impl, File('h')._impl, File('i')._impl ] self.appconfig_outputbox = ['j', 'k', 'l'] self.appmasterconfig = StandardJobConfig( inputbox=self.appconfig_inputbox, outputbox=self.appconfig_outputbox) self._rthandler = ExeDiracRTHandler() self.job_list = [] # setup test app jobs self.j_command = Job(application=Executable(exe='ls'), backend=Dirac(), inputsandbox=self.inputsandbox, outputfiles=self.outputsandbox) self.j_command.prepare() self.job_list.append(self.j_command) ## j_app_qualified = Job(application=Executable(exe='/bin/echo', args=['Hello', 'World']), backend=Dirac(), inputsandbox=self.inputsandbox, outputfiles=self.outputsandbox) j_app_qualified.prepare() self.job_list.append(j_app_qualified) ## f = tempfile.NamedTemporaryFile(mode='w') self.tmp_filename = os.path.basename(f.name) f.write('#!/bin/bash\necho 123') j_file = Job(application=Executable(exe=File(f.name)), backend=Dirac(), inputsandbox=self.inputsandbox, outputfiles=self.outputsandbox) j_file.prepare() f.close() self.job_list.append(j_file)
class TestExeDiracRTHandler(GangaGPITestCase): def setUp(self): # initial setup self.inputsandbox = [File('a'), File('b'), File('c')] self.outputsandbox = ['d', 'e', 'f'] self.appconfig_inputbox = [ File('g')._impl, File('h')._impl, File('i')._impl] self.appconfig_outputbox = ['j', 'k', 'l'] self.appmasterconfig = StandardJobConfig(inputbox=self.appconfig_inputbox, outputbox=self.appconfig_outputbox) self._rthandler = ExeDiracRTHandler() self.job_list = [] # setup test app jobs self.j_command = Job(application=Executable(exe='ls'), backend=Dirac(), inputsandbox=self.inputsandbox, outputfiles=self.outputsandbox) self.j_command.prepare() self.job_list.append(self.j_command) ## j_app_qualified = Job(application=Executable(exe='/bin/echo', args=['Hello', 'World']), backend=Dirac(), inputsandbox=self.inputsandbox, outputfiles=self.outputsandbox) j_app_qualified.prepare() self.job_list.append(j_app_qualified) ## f = tempfile.NamedTemporaryFile(mode='w') self.tmp_filename = os.path.basename(f.name) f.write('#!/bin/bash\necho 123') j_file = Job(application=Executable(exe=File(f.name)), backend=Dirac(), inputsandbox=self.inputsandbox, outputfiles=self.outputsandbox) j_file.prepare() f.close() self.job_list.append(j_file) ## def test_master_prepare(self): # setup failing app jobs g = tempfile.NamedTemporaryFile(mode='w') j_fail = Job(application=Executable(exe=File(g.name))) j_fail.prepare() g.close() # file now gone os.remove(os.path.join( get_share_path(j_fail.application._impl), os.path.split(g.name)[1])) ## j_prep_fail = Job(application=Executable(exe='ls')) # Start the testing for each app type for app in (j.application._impl for j in self.job_list): # check its a known app type self.assertTrue(isinstance(app.exe, str) or isinstance( app.exe, File._impl), 'Unknown app.exe type! %s' % type(app.exe)) # run the method we are testing jobconfig = self._rthandler.master_prepare( app, self.appmasterconfig) # check the return value is of the right type self.assertTrue(isinstance(jobconfig, StandardJobConfig), 'Expected a StandardJobConfig object returned. Instead got %s' % repr(jobconfig)) # create sets from the text string file names from the inputbox and # outputbox ipb = set(f.name for f in jobconfig.inputbox) opb = set(jobconfig.outputbox) # check that inputbox and outputbox contain only unique elements self.assertEqual(len(ipb), len( jobconfig.inputbox), 'Returned inputsandbox did not contain only unique elements') self.assertEqual(len(opb), len( jobconfig.outputbox), 'Returned outputsandbox did not contain only unique elements') # find the difference between the in/outputbox and those from the # defined job in/outputsandbox and appconfig_in/outputbox idiff = ipb.symmetric_difference( set([f.name for f in self.inputsandbox] + [f.name for f in self.appconfig_inputbox])) # added __postprocesslocations__ odiff = opb.symmetric_difference( set(self.outputsandbox + self.appconfig_outputbox + ['__postprocesslocations__'])) # expect that things placed in the sharedir on preparation will # feature in idiff so check and remove them for root, dirs, files in os.walk(get_share_path(app)): if files: qualified_files = set( [os.path.join(root, f) for f in files]) self.assertTrue(qualified_files.issubset( idiff), 'Could not find the following prepared file(s) in jobconfig.inputbox: %s' % repr(qualified_files.difference(idiff))) # once checked that they exist in the idiff then remove # them for ultimate check next idiff.difference_update(qualified_files) # check that no extra files, i.e. those not from the # job.in/outputsandbox or appconfig_in/outputbox or sharedir are # present self.assertEqual(idiff, set( ), 'jobconfig.inputbox != job.inputsandbox + appconfig.inputbox + prepared_sharedir_files: sym_diff = %s' % idiff) self.assertEqual(odiff, set( ), 'jobconfig.outputbox != job.outputsandbox + appconfig.outputbox: sym_diff = %s' % odiff) # check that the proper exception is raised in case of the exe file not # existing self.assertRaises(ApplicationConfigurationError, self._rthandler.master_prepare, j_fail.application._impl, self.appmasterconfig, msg="Checking that Exception raised if file doesn't exist") # check that the proper exception is raised in case of the app not # being prepared. self.assertRaises(GangaException, self._rthandler.master_prepare, j_prep_fail.application._impl, self.appmasterconfig, msg="Checking exception raised if app not prepared") def test_prepare(self): appsubconfig = StandardJobConfig(inputbox=[File('file1.txt')._impl, File('file2.txt')._impl], outputbox=['file3.txt', 'file4.txt']) jobmasterconfig = StandardJobConfig(inputbox=[File('file5.txt')._impl, File('file6.txt')._impl], outputbox=['file7.txt', 'file8.txt']) # Start the testing for each app type for app in (j.application._impl for j in self.job_list): jobsubconfig = self._rthandler.prepare( app, appsubconfig, self.appmasterconfig, jobmasterconfig) # create sets from the text string file names from the inputbox and # outputbox ipb = set(f.name for f in jobsubconfig.inputbox) opb = set(jobsubconfig.outputbox) # check that inputbox and outputbox contain only unique elements self.assertEqual(len(ipb), len( jobsubconfig.inputbox), 'Returned inputsandbox did not contain only unique elements') self.assertEqual(len(opb), len( jobsubconfig.outputbox), 'Returned outputsandbox did not contain only unique elements') # find the difference between the in/outputbox and those from the # defined job in/outputsandbox and appconfig_in/outputbox idiff = ipb.symmetric_difference( set([f.name for f in appsubconfig.inputbox] + ['exe-script.py'])) odiff = opb.symmetric_difference( set(appsubconfig.outputbox + jobmasterconfig.outputbox)) if isinstance(app.exe, File._impl): fname = os.path.join(get_share_path(app), self.tmp_filename) self.assertTrue( fname in idiff, "Couldn't find the exe file in inputsandbox") # once checked that they exist in the idiff then remove them # for ultimate check next idiff.remove(fname) # check that no extra files, i.e. those not from the # job.in/outputsandbox or appconfig_in/outputbox or sharedir are # present self.assertEqual(idiff, set( ), 'jobsubconfig.inputbox != appsubconfig.inputbox + exe-script.py + exe file: sym_diff = %s' % idiff) self.assertEqual(odiff, set( ), 'jobsubconfig.outputbox != appsubconfig.outputbox + jobmasterconfig.outputbox: sym_diff = %s' % odiff) script = \ """# dirac job created by ganga from DIRAC.Core.Base.Script import parseCommandLine parseCommandLine() from DIRAC.Interfaces.API.Dirac import Dirac from DIRAC.Interfaces.API.Job import Job dirac = Dirac() j = Job() # default commands added by ganga j.setName('{Ganga_Executable_(###JOB_ID###)}') j.setExecutable('exe-script.py','','Ganga_Executable.log') j.setInputSandbox(##INPUT_SANDBOX##) j.setOutputSandbox(['file4.txt', 'file3.txt', 'file8.txt', 'file7.txt']) # <-- user settings j.setCPUTime(172800) # user settings --> # diracOpts added by user # submit the job to dirac result = dirac.submit(j) output(result)""" self.assertEqual(jobsubconfig.exe, script.replace( '###JOB_ID###', app._getParent().fqid), 'Dirac API script does not match, see diff below:\n' + '\n'.join(difflib.unified_diff(jobsubconfig.exe.splitlines(), script.replace( '###JOB_ID###', app._getParent().fqid).splitlines(), fromfile='Coming from prepare method', tofile='What the test expected'))) # NEED SOME CHECK THAT THE EXE SCRIPT IS GENERATED PROPERLY def test_exe_script_template(self): script_template = """#!/usr/bin/env python '''Script to run Executable application''' from os import system, environ, pathsep, getcwd import sys # Main if __name__ == '__main__': environ['PATH'] = getcwd() + (pathsep + environ['PATH']) rc = (system('###COMMAND###')/256) ###OUTPUTFILESINJECTEDCODE### sys.exit(rc) """ # check that exe_script_template matches above self.assertEqual(script_template.replace(" ", ""), exe_script_template().replace( " ", ""), 'Returned template doesn\'t match expectation.')