def _create_kickoff_script(self, pav_cfg, test_obj): """Function to accept a list of lines and generate a script that is then submitted to the scheduler. :param pavilion.test_config.TestRun test_obj: """ header = self._get_kickoff_script_header(test_obj) script = scriptcomposer.ScriptComposer(header=header) script.comment("Redirect all output to kickoff.log") script.command("exec >{} 2>&1".format(test_obj.path / 'kickoff.log')) # Make sure the pavilion spawned env_changes = { 'PATH': '{}:${{PATH}}'.format(pav_cfg.pav_root / 'bin'), 'PAV_CONFIG_FILE': str(pav_cfg.pav_cfg_file), } if 'PAV_CONFIG_DIR' in os.environ: env_changes['PAV_CONFIG_DIR'] = os.environ['PAV_CONFIG_DIR'] script.env_change(env_changes) # Run Kickoff Env setup commands for command in pav_cfg.env_setup: script.command(command) # Run the test via pavilion script.command('pav _run {t.id}'.format(t=test_obj)) path = self._kickoff_script_path(test_obj) script.write(path) return path
def test_scriptComposer(self): """Testing ScriptComposer class variable setting.""" # Testing valid uses. # Testing initialization defaults. composer = scriptcomposer.ScriptComposer() self.assertEqual(composer.header.shebang, '#!/bin/bash') self.assertEqual(composer.header.scheduler_headers, []) # Testing individual assignment test_header_shell = "/usr/env/python" test_header_scheduler = OrderedDict() test_header_scheduler['-G'] = 'pam' test_header_scheduler['-N'] = 'fam' composer.newline() composer.command(['taco', 'burrito', 'nachos']) test_details_path = 'testPath' test_details_group = 'groupies' test_details_perms = 0o543 composer.header.shebang = test_header_shell composer.header.scheduler_headers = test_header_scheduler self.assertEqual(composer.header.shebang, test_header_shell) self.assertEqual(composer.header.scheduler_headers, test_header_scheduler) composer = scriptcomposer.ScriptComposer() self.assertEqual(composer.header.shebang, '#!/bin/bash') self.assertEqual(composer.header.scheduler_headers, []) # Testing object assignment. header = scriptcomposer.ScriptHeader( shebang=test_header_shell, scheduler_headers=test_header_scheduler) composer.header = header self.assertEqual(composer.header.shebang, test_header_shell) self.assertEqual(composer.header.scheduler_headers, test_header_scheduler)
def _write_script(self, path, config, sys_vars): """Write a build or run script or template. The formats for each are identical. :param str path: Path to the template file to write. :param dict config: Configuration dictionary for the script file. :return: """ if sys_vars is None: raise RuntimeError("Trying to write script without sys_vars " "in test '{}'.".format(self.id)) script = scriptcomposer.ScriptComposer( details=scriptcomposer.ScriptDetails( path=path, group=self._pav_cfg.shared_group, )) pav_lib_bash = self._pav_cfg.pav_root / 'bin' / 'pav-lib.bash' # If we include this directly, it breaks build hashing. script.comment('The first (and only) argument of the build script is ' 'the test id.') script.env_change({'TEST_ID': '$1'}) script.command('source {}'.format(pav_lib_bash)) modules = config.get('modules', []) if modules: script.newline() script.comment( 'Perform module related changes to the environment.') for module in config.get('modules', []): script.module_change(module, sys_vars) env = config.get('env', {}) if env: script.newline() script.comment("Making any environment changes needed.") script.env_change(config.get('env', {})) script.newline() cmds = config.get('cmds', []) if cmds: script.comment("Perform the sequence of test commands.") for line in config.get('cmds', []): for split_line in line.split('\n'): script.command(split_line) else: script.comment('No commands given for this script.') script.write()
def test_writeScript(self): """Testing the writeScript function of the ScriptComposer class.""" testHeaderShell = "/usr/env/python" testHeaderScheduler = ['-G pam', '-N fam'] testDetailsPath = self.pav_cfg.working_dir / 'testPath' testDetailsGroup = self._other_group() testDetailsPerms = 0o760 testComposer = scriptcomposer.ScriptComposer() testComposer.header.shell_path = testHeaderShell testComposer.header.scheduler_headers = testHeaderScheduler testComposer.details.path = testDetailsPath testComposer.details.group = testDetailsGroup testComposer.details.perms = testDetailsPerms testDir = os.getcwd() testComposer.write() self.assertTrue(testDetailsPath.exists()) with testDetailsPath.open() as testFile: testLines = testFile.readlines() for i in range(0, len(testLines)): testLines[i] = testLines[i].strip() self.assertEqual(testLines[0], "#!/usr/env/python") self.assertEqual(testLines[1], "# -G pam") self.assertEqual(testLines[2], "# -N fam") self.assertEqual(testLines[3], "") self.assertEqual(testLines[4], "") self.assertEqual(len(testLines), 5) testStat = testDetailsPath.stat() expectedStat = stat.S_IFREG + stat.S_IRWXU + stat.S_IRGRP + stat.S_IWGRP self.assertEqual(testStat.st_mode, expectedStat) testGID = testStat.st_gid testGID = grp.getgrgid(testGID).gr_name self.assertEqual(testGID, testDetailsGroup) testDetailsPath.unlink()
def _write_script(self, path, config): """Write a build or run script or template. The formats for each are identical. :param str path: Path to the template file to write. :param dict config: Configuration dictionary for the script file. :return: """ script = scriptcomposer.ScriptComposer( details=scriptcomposer.ScriptDetails( path=path, group=self._pav_cfg.shared_group, )) pav_lib_bash = os.path.join(self._pav_cfg.pav_root, 'bin', 'pav-lib.bash') script.comment('The following is added to every test build and ' 'run script.') script.env_change({'TEST_ID': '{}'.format(self.id)}) script.command('source {}'.format(pav_lib_bash)) modules = config.get('modules', []) if modules: script.newline() script.comment('Perform module related changes to the environment.') for module in config.get('modules', []): script.module_change(module, self._pav_cfg.sys_vars) env = config.get('env', {}) if env: script.newline() script.comment("Making any environment changes needed.") script.env_change(config.get('env', {})) script.newline() cmds = config.get('cmds', []) if cmds: script.comment("Perform the sequence of test commands.") for line in config.get('cmds', []): for split_line in line.split('\n'): script.command(split_line) else: script.comment('No commands given for this script.') script.write()
def test_writeScript(self): """Testing the writeScript function of the ScriptComposer class.""" testHeaderShell = "/usr/env/python" testHeaderScheduler = ['-G pam', '-N fam'] path = self.pav_cfg.working_dir / 'testPath' testComposer = scriptcomposer.ScriptComposer() testComposer.header.shebang = testHeaderShell testComposer.header.scheduler_headers = testHeaderScheduler testComposer.write(path) self.assertTrue(path.exists()) with path.open() as testFile: testLines = testFile.readlines() for i in range(0, len(testLines)): testLines[i] = testLines[i].strip() self.assertEqual(testLines[0], "#!/usr/env/python") self.assertEqual(testLines[1], "# -G pam") self.assertEqual(testLines[2], "# -N fam") self.assertEqual(testLines[3], "") self.assertEqual(testLines[4], "") self.assertEqual(len(testLines), 5) testStat = path.stat() umask = os.umask(0) os.umask(umask) # Default file permissions. expectedStat = (0o100666 & ~umask) | stat.S_IXGRP | stat.S_IXUSR self.assertEqual(oct(testStat.st_mode), oct(expectedStat)) path.unlink()
def _write_kick_off_script(self, test): """Function to accept a list of lines and generate a script that is then submitted to the scheduler. """ header = self._get_kick_off_header(test) script = scriptcomposer.ScriptComposer( header=header, details=scriptcomposer.ScriptDetails( path=os.path.join(test.path, 'kickoff.{}'.format(self.KICKOFF_SCRIPT_EXT)) ), ) script.newline() script.comment("Within the allocation, run the command.") script.command(test.run_cmd()) script.write() return script.details.path
def _write_script(self, path, config): """Write a build or run script or template. The formats for each are identical. :param Path path: Path to the template file to write. :param dict config: Configuration dictionary for the script file. :return: """ script = scriptcomposer.ScriptComposer( details=scriptcomposer.ScriptDetails( path=path, group=self._pav_cfg.shared_group, )) verbose = config.get('verbose', 'false').lower() == 'true' if verbose: script.comment('# Echoing all commands to log.') script.command('set -v') script.newline() pav_lib_bash = self._pav_cfg.pav_root / 'bin' / 'pav-lib.bash' # If we include this directly, it breaks build hashing. script.comment('The first (and only) argument of the build script is ' 'the test id.') script.env_change({ 'TEST_ID': '${1:-0}', # Default to test id 0 if one isn't given. 'PAV_CONFIG_FILE': self._pav_cfg['pav_cfg_file'] }) script.command('source {}'.format(pav_lib_bash)) if config.get('preamble', []): script.newline() script.comment('Preamble commands') for cmd in config['preamble']: script.command(cmd) modules = config.get('modules', []) if modules: script.newline() script.comment( 'Perform module related changes to the environment.') for module in config.get('modules', []): script.module_change(module, self.var_man) env = config.get('env', {}) if env: script.newline() script.comment("Making any environment changes needed.") script.env_change(config.get('env', {})) if verbose: script.newline() script.comment('List all the module modules for posterity') script.command("module -t list") script.newline() script.comment('Output the environment for posterity') script.command("declare -p") script.newline() cmds = config.get('cmds', []) if cmds: script.comment("Perform the sequence of test commands.") for line in config.get('cmds', []): for split_line in line.split('\n'): script.command(split_line) else: script.comment('No commands given for this script.') script.write()
def test_writeScript_2( self ): """Testing the writeScript function of the ScriptComposer class.""" testHeaderShell = "/usr/env/python" testHeaderScheduler = OrderedDict() testHeaderScheduler[ '-G' ] = 'pam' testHeaderScheduler[ '-N' ] = 'fam' testDetailsName = 'testName' testDetailsType = 'batch' testDetailsUser = os.environ['USER'] testDetailsGroup = self._other_group() testDetailsOP = 7 testDetailsGP = 6 testDetailsWP = 0 testComposer = scriptcomposer.ScriptComposer() testComposer.header.shell_path = testHeaderShell testComposer.header.scheduler_macros = testHeaderScheduler testComposer.body.name = testDetailsName testComposer.body.script_type = testDetailsType testComposer.body.user = testDetailsUser testComposer.body.group = testDetailsGroup testComposer.body.owner_perms = testDetailsOP testComposer.body.group_perms = testDetailsGP testComposer.body.world_perms = testDetailsWP testDir = os.getcwd() testComposer.write_script(testDir) self.assertTrue( os.path.isfile( os.path.join( testDir, testDetailsName + '.batch' ) ) ) testFile = open(os.path.join( testDir, testDetailsName+'.batch'), 'r' ) testLines = testFile.readlines() testFile.close() for i in range(0, len( testLines ) ): testLines[i] = testLines[i].strip() self.assertEqual( testLines[0], "#!/usr/env/python" ) self.assertEqual( testLines[1], "# -G pam" ) self.assertEqual( testLines[2], "# -N fam" ) self.assertEqual( testLines[3], "" ) self.assertEqual( len(testLines), 4 ) testStat = os.stat( os.path.join(testDir, testDetailsName + '.batch') ) expectedStat=stat.S_IFREG + stat.S_IRWXU + stat.S_IRGRP + stat.S_IWGRP self.assertEqual( testStat.st_mode, expectedStat ) testUID = testStat.st_uid testGID = testStat.st_gid testUID = pwd.getpwuid( testUID ).pw_name testGID = grp.getgrgid( testGID ).gr_name self.assertEqual( testUID, testDetailsUser ) self.assertEqual( testGID, testDetailsGroup ) os.remove( os.path.join( testDir, testDetailsName + '.batch' ) )
def test_scriptComposer( self ): """Testing ScriptComposer class variable setting.""" # Testing valid uses. # Testing initialization defaults. testComposer = scriptcomposer.ScriptComposer() self.assertIsInstance(testComposer.header, scriptcomposer.ScriptHeader) self.assertIsInstance(testComposer.body, scriptcomposer.ScriptBody) self.assertIsNone( testComposer.header.shell_path ) self.assertIsNone( testComposer.header.scheduler_macros ) self.assertEqual(testComposer.body.script_type, "bash") self.assertEqual(testComposer.body.user, os.environ['USER']) self.assertEqual(testComposer.body.group, os.environ['USER']) self.assertEqual(testComposer.body.owner_perms, 7) self.assertEqual(testComposer.body.group_perms, 5) self.assertEqual(testComposer.body.world_perms, 0) # Testing individual assignment testHeaderShell = "/usr/env/python" testHeaderScheduler = OrderedDict() testHeaderScheduler[ '-G' ] = 'pam' testHeaderScheduler[ '-N' ] = 'fam' testComposer.newline() testComposer.command(['taco', 'burrito', 'nachos']) testDetailsName = 'testName' testDetailsType = 'batch' testDetailsUser = '******' testDetailsGroup = 'groupies' testDetailsOP = 1 testDetailsGP = 2 testDetailsWP = 3 testComposer.header.shell_path = testHeaderShell testComposer.header.scheduler_macros = testHeaderScheduler testComposer.body.name = testDetailsName testComposer.body.script_type = testDetailsType testComposer.body.user = testDetailsUser testComposer.body.group = testDetailsGroup testComposer.body.owner_perms = testDetailsOP testComposer.body.group_perms = testDetailsGP testComposer.body.world_perms = testDetailsWP self.assertEqual( testComposer.header.shell_path, testHeaderShell ) self.assertEqual( testComposer.header.scheduler_macros, testHeaderScheduler ) self.assertEqual(testComposer.body.name, testDetailsName) self.assertEqual(testComposer.body.script_type, testDetailsType) self.assertEqual(testComposer.body.user, testDetailsUser) self.assertEqual(testComposer.body.group, testDetailsGroup) self.assertEqual(testComposer.body.owner_perms, testDetailsOP) self.assertEqual(testComposer.body.group_perms, testDetailsGP) self.assertEqual(testComposer.body.world_perms, testDetailsWP) # Testing reset. testComposer.reset() self.assertIsNone( testComposer.header.shell_path ) self.assertIsNone( testComposer.header.scheduler_macros ) self.assertEqual(testComposer.body.script_type, "bash") self.assertEqual(testComposer.body.user, os.environ['USER']) self.assertEqual(testComposer.body.group, os.environ['USER']) self.assertEqual(testComposer.body.owner_perms, 7) self.assertEqual(testComposer.body.group_perms, 5) self.assertEqual(testComposer.body.world_perms, 0) # Testing object assignment. testHeaderObj = scriptcomposer.ScriptHeader(shell_path=testHeaderShell, scheduler_macros= testHeaderScheduler) testDetailsObj = scriptcomposer.ScriptBody(name=testDetailsName, script_type= testDetailsType, user=testDetailsUser, group=testDetailsGroup, owner_perms= testDetailsOP, group_perms= testDetailsGP, world_perms= testDetailsWP) testComposer.header = testHeaderObj testComposer.body = testDetailsObj self.assertEqual( testComposer.header.shell_path, testHeaderShell ) self.assertEqual( testComposer.header.scheduler_macros, testHeaderScheduler ) self.assertEqual(testComposer.body.name, testDetailsName) self.assertEqual(testComposer.body.script_type, testDetailsType) self.assertEqual(testComposer.body.user, testDetailsUser) self.assertEqual(testComposer.body.group, testDetailsGroup) self.assertEqual(testComposer.body.owner_perms, testDetailsOP) self.assertEqual(testComposer.body.group_perms, testDetailsGP) self.assertEqual(testComposer.body.world_perms, testDetailsWP)
def test_scriptComposer(self): """Testing ScriptComposer class variable setting.""" # Testing valid uses. # Testing initialization defaults. composer = scriptcomposer.ScriptComposer() self.assertIsInstance(composer.header, scriptcomposer.ScriptHeader) self.assertIsInstance(composer.details, scriptcomposer.ScriptDetails) self.assertEqual(composer.header.shell_path, '#!/bin/bash') self.assertEqual(composer.header.scheduler_headers, []) self.assertEqual(composer.details.group, utils.get_login()) self.assertEqual(composer.details.perms, oct(0o770)) # Testing individual assignment test_header_shell = "/usr/env/python" test_header_scheduler = OrderedDict() test_header_scheduler['-G'] = 'pam' test_header_scheduler['-N'] = 'fam' composer.newline() composer.command(['taco', 'burrito', 'nachos']) test_details_path = 'testPath' test_details_group = 'groupies' test_details_perms = 0o543 composer.header.shell_path = test_header_shell composer.header.scheduler_headers = test_header_scheduler composer.details.path = test_details_path composer.details.group = test_details_group composer.details.perms = test_details_perms self.assertEqual(composer.header.shell_path, test_header_shell) self.assertEqual(composer.header.scheduler_headers, test_header_scheduler) self.assertEqual(composer.details.path, Path(test_details_path)) self.assertEqual(composer.details.group, test_details_group) self.assertEqual(composer.details.perms, oct(test_details_perms)) composer = scriptcomposer.ScriptComposer() self.assertEqual(composer.header.shell_path, '#!/bin/bash') self.assertEqual(composer.header.scheduler_headers, []) self.assertEqual(composer.details.group, utils.get_login()) self.assertEqual(composer.details.perms, oct(0o770)) # Testing object assignment. header = scriptcomposer.ScriptHeader( shell_path=test_header_shell, scheduler_headers=test_header_scheduler) testDetailsObj = scriptcomposer.ScriptDetails(path=test_details_path, group=test_details_group, perms=test_details_perms) composer.header = header composer.details = testDetailsObj self.assertEqual(composer.header.shell_path, test_header_shell) self.assertEqual(composer.header.scheduler_headers, test_header_scheduler) self.assertEqual(composer.details.path, Path(test_details_path)) self.assertEqual(composer.details.group, test_details_group) self.assertEqual(composer.details.perms, oct(test_details_perms))
def _write_script(self, stype, path, config): """Write a build or run script or template. The formats for each are mostly identical. :param str stype: The type of script (run or build). :param Path path: Path to the template file to write. :param dict config: Configuration dictionary for the script file. :return: """ script = scriptcomposer.ScriptComposer() verbose = config.get('verbose', 'false').lower() == 'true' if verbose: script.comment('# Echoing all commands to log.') script.command('set -v') script.newline() pav_lib_bash = self._pav_cfg.pav_root / 'bin' / 'pav-lib.bash' # If we include this directly, it breaks build hashing. script.comment('The first (and only) argument of the build script is ' 'the test id.') script.env_change({ 'TEST_ID': '${1:-0}', # Default to test id 0 if one isn't given. 'PAV_CONFIG_FILE': self._pav_cfg['pav_cfg_file'] }) script.command('source {}'.format(pav_lib_bash)) if config.get('preamble', []): script.newline() script.comment('Preamble commands') for cmd in config['preamble']: script.command(cmd) if stype == 'build' and not self.build_local: script.comment('To be built in an allocation.') modules = config.get('modules', []) if modules: script.newline() script.comment( 'Perform module related changes to the environment.') for module in config.get('modules', []): script.module_change(module, self.var_man) env = config.get('env', {}) if env: script.newline() script.comment("Making any environment changes needed.") script.env_change(config.get('env', {})) if verbose: script.newline() script.comment('List all the module modules for posterity') script.command("module -t list") script.newline() script.comment('Output the environment for posterity') script.command("declare -p") if self.spack_enabled(): spack_commands = config.get('spack', {}) install_packages = spack_commands.get('install', []) load_packages = spack_commands.get('load', []) script.newline() script.comment('Source spack setup script.') script.command('source {}/share/spack/setup-env.sh'.format( self._pav_cfg.get('spack_path'))) script.newline() script.command('spack env deactivate &>/dev/null') script.comment('Activate spack environment.') script.command('spack env activate -d .') script.command('if [ -z $SPACK_ENV ]; then exit 1; fi') if install_packages: script.newline() script.comment('Install spack packages.') for package in install_packages: script.command( 'spack install -v --fail-fast {} || exit 1'.format( package)) if load_packages: script.newline() script.comment('Load spack packages.') for package in load_packages: script.command('spack load {} || exit 1'.format(package)) script.newline() cmds = config.get('cmds', []) if cmds: script.comment("Perform the sequence of test commands.") for line in config.get('cmds', []): for split_line in line.split('\n'): script.command(split_line) else: script.comment('No commands given for this script.') with PermissionsManager(path, self.group, self.umask): script.write(path)