def _create_kickoff_script(self, pav_cfg, test_obj):
"""Function to accept a list of lines and generate a script that is
then submitted to the scheduler.
:param pavilion.test_config.TestRun test_obj:
"""
header = self._get_kickoff_script_header(test_obj)
script = scriptcomposer.ScriptComposer(header=header)
script.comment("Redirect all output to kickoff.log")
script.command("exec >{} 2>&1".format(test_obj.path / 'kickoff.log'))
# Make sure the pavilion spawned
env_changes = {
'PATH': '{}:${{PATH}}'.format(pav_cfg.pav_root / 'bin'),
'PAV_CONFIG_FILE': str(pav_cfg.pav_cfg_file),
}
if 'PAV_CONFIG_DIR' in os.environ:
env_changes['PAV_CONFIG_DIR'] = os.environ['PAV_CONFIG_DIR']
script.env_change(env_changes)
# Run Kickoff Env setup commands
for command in pav_cfg.env_setup:
script.command(command)
# Run the test via pavilion
script.command('pav _run {t.id}'.format(t=test_obj))
path = self._kickoff_script_path(test_obj)
script.write(path)
return path
def test_scriptComposer(self):
"""Testing ScriptComposer class variable setting."""
# Testing valid uses.
# Testing initialization defaults.
composer = scriptcomposer.ScriptComposer()
self.assertEqual(composer.header.shebang, '#!/bin/bash')
self.assertEqual(composer.header.scheduler_headers, [])
# Testing individual assignment
test_header_shell = "/usr/env/python"
test_header_scheduler = OrderedDict()
test_header_scheduler['-G'] = 'pam'
test_header_scheduler['-N'] = 'fam'
composer.newline()
composer.command(['taco', 'burrito', 'nachos'])
test_details_path = 'testPath'
test_details_group = 'groupies'
test_details_perms = 0o543
composer.header.shebang = test_header_shell
composer.header.scheduler_headers = test_header_scheduler
self.assertEqual(composer.header.shebang, test_header_shell)
self.assertEqual(composer.header.scheduler_headers,
test_header_scheduler)
composer = scriptcomposer.ScriptComposer()
self.assertEqual(composer.header.shebang, '#!/bin/bash')
self.assertEqual(composer.header.scheduler_headers, [])
# Testing object assignment.
header = scriptcomposer.ScriptHeader(
shebang=test_header_shell, scheduler_headers=test_header_scheduler)
composer.header = header
self.assertEqual(composer.header.shebang, test_header_shell)
self.assertEqual(composer.header.scheduler_headers,
test_header_scheduler)
def _write_script(self, path, config, sys_vars):
"""Write a build or run script or template. The formats for each are
identical.
:param str path: Path to the template file to write.
:param dict config: Configuration dictionary for the script file.
:return:
"""
if sys_vars is None:
raise RuntimeError("Trying to write script without sys_vars "
"in test '{}'.".format(self.id))
script = scriptcomposer.ScriptComposer(
details=scriptcomposer.ScriptDetails(
path=path,
group=self._pav_cfg.shared_group,
))
pav_lib_bash = self._pav_cfg.pav_root / 'bin' / 'pav-lib.bash'
# If we include this directly, it breaks build hashing.
script.comment('The first (and only) argument of the build script is '
'the test id.')
script.env_change({'TEST_ID': '$1'})
script.command('source {}'.format(pav_lib_bash))
modules = config.get('modules', [])
if modules:
script.newline()
script.comment(
'Perform module related changes to the environment.')
for module in config.get('modules', []):
script.module_change(module, sys_vars)
env = config.get('env', {})
if env:
script.newline()
script.comment("Making any environment changes needed.")
script.env_change(config.get('env', {}))
script.newline()
cmds = config.get('cmds', [])
if cmds:
script.comment("Perform the sequence of test commands.")
for line in config.get('cmds', []):
for split_line in line.split('\n'):
script.command(split_line)
else:
script.comment('No commands given for this script.')
script.write()
def test_writeScript(self):
"""Testing the writeScript function of the ScriptComposer class."""
testHeaderShell = "/usr/env/python"
testHeaderScheduler = ['-G pam', '-N fam']
testDetailsPath = self.pav_cfg.working_dir / 'testPath'
testDetailsGroup = self._other_group()
testDetailsPerms = 0o760
testComposer = scriptcomposer.ScriptComposer()
testComposer.header.shell_path = testHeaderShell
testComposer.header.scheduler_headers = testHeaderScheduler
testComposer.details.path = testDetailsPath
testComposer.details.group = testDetailsGroup
testComposer.details.perms = testDetailsPerms
testDir = os.getcwd()
testComposer.write()
self.assertTrue(testDetailsPath.exists())
with testDetailsPath.open() as testFile:
testLines = testFile.readlines()
for i in range(0, len(testLines)):
testLines[i] = testLines[i].strip()
self.assertEqual(testLines[0], "#!/usr/env/python")
self.assertEqual(testLines[1], "# -G pam")
self.assertEqual(testLines[2], "# -N fam")
self.assertEqual(testLines[3], "")
self.assertEqual(testLines[4], "")
self.assertEqual(len(testLines), 5)
testStat = testDetailsPath.stat()
expectedStat = stat.S_IFREG + stat.S_IRWXU + stat.S_IRGRP + stat.S_IWGRP
self.assertEqual(testStat.st_mode, expectedStat)
testGID = testStat.st_gid
testGID = grp.getgrgid(testGID).gr_name
self.assertEqual(testGID, testDetailsGroup)
testDetailsPath.unlink()
def _write_script(self, path, config):
"""Write a build or run script or template. The formats for each are
identical.
:param str path: Path to the template file to write.
:param dict config: Configuration dictionary for the script file.
:return:
"""
script = scriptcomposer.ScriptComposer(
details=scriptcomposer.ScriptDetails(
path=path,
group=self._pav_cfg.shared_group,
))
pav_lib_bash = os.path.join(self._pav_cfg.pav_root,
'bin', 'pav-lib.bash')
script.comment('The following is added to every test build and '
'run script.')
script.env_change({'TEST_ID': '{}'.format(self.id)})
script.command('source {}'.format(pav_lib_bash))
modules = config.get('modules', [])
if modules:
script.newline()
script.comment('Perform module related changes to the environment.')
for module in config.get('modules', []):
script.module_change(module, self._pav_cfg.sys_vars)
env = config.get('env', {})
if env:
script.newline()
script.comment("Making any environment changes needed.")
script.env_change(config.get('env', {}))
script.newline()
cmds = config.get('cmds', [])
if cmds:
script.comment("Perform the sequence of test commands.")
for line in config.get('cmds', []):
for split_line in line.split('\n'):
script.command(split_line)
else:
script.comment('No commands given for this script.')
script.write()
def test_writeScript(self):
"""Testing the writeScript function of the ScriptComposer class."""
testHeaderShell = "/usr/env/python"
testHeaderScheduler = ['-G pam', '-N fam']
path = self.pav_cfg.working_dir / 'testPath'
testComposer = scriptcomposer.ScriptComposer()
testComposer.header.shebang = testHeaderShell
testComposer.header.scheduler_headers = testHeaderScheduler
testComposer.write(path)
self.assertTrue(path.exists())
with path.open() as testFile:
testLines = testFile.readlines()
for i in range(0, len(testLines)):
testLines[i] = testLines[i].strip()
self.assertEqual(testLines[0], "#!/usr/env/python")
self.assertEqual(testLines[1], "# -G pam")
self.assertEqual(testLines[2], "# -N fam")
self.assertEqual(testLines[3], "")
self.assertEqual(testLines[4], "")
self.assertEqual(len(testLines), 5)
testStat = path.stat()
umask = os.umask(0)
os.umask(umask)
# Default file permissions.
expectedStat = (0o100666 & ~umask) | stat.S_IXGRP | stat.S_IXUSR
self.assertEqual(oct(testStat.st_mode), oct(expectedStat))
path.unlink()
def _write_kick_off_script(self, test):
"""Function to accept a list of lines and generate a script that is
then submitted to the scheduler.
"""
header = self._get_kick_off_header(test)
script = scriptcomposer.ScriptComposer(
header=header,
details=scriptcomposer.ScriptDetails(
path=os.path.join(test.path,
'kickoff.{}'.format(self.KICKOFF_SCRIPT_EXT))
),
)
script.newline()
script.comment("Within the allocation, run the command.")
script.command(test.run_cmd())
script.write()
return script.details.path
def _write_script(self, path, config):
"""Write a build or run script or template. The formats for each are
identical.
:param Path path: Path to the template file to write.
:param dict config: Configuration dictionary for the script file.
:return:
"""
script = scriptcomposer.ScriptComposer(
details=scriptcomposer.ScriptDetails(
path=path,
group=self._pav_cfg.shared_group,
))
verbose = config.get('verbose', 'false').lower() == 'true'
if verbose:
script.comment('# Echoing all commands to log.')
script.command('set -v')
script.newline()
pav_lib_bash = self._pav_cfg.pav_root / 'bin' / 'pav-lib.bash'
# If we include this directly, it breaks build hashing.
script.comment('The first (and only) argument of the build script is '
'the test id.')
script.env_change({
'TEST_ID': '${1:-0}', # Default to test id 0 if one isn't given.
'PAV_CONFIG_FILE': self._pav_cfg['pav_cfg_file']
})
script.command('source {}'.format(pav_lib_bash))
if config.get('preamble', []):
script.newline()
script.comment('Preamble commands')
for cmd in config['preamble']:
script.command(cmd)
modules = config.get('modules', [])
if modules:
script.newline()
script.comment(
'Perform module related changes to the environment.')
for module in config.get('modules', []):
script.module_change(module, self.var_man)
env = config.get('env', {})
if env:
script.newline()
script.comment("Making any environment changes needed.")
script.env_change(config.get('env', {}))
if verbose:
script.newline()
script.comment('List all the module modules for posterity')
script.command("module -t list")
script.newline()
script.comment('Output the environment for posterity')
script.command("declare -p")
script.newline()
cmds = config.get('cmds', [])
if cmds:
script.comment("Perform the sequence of test commands.")
for line in config.get('cmds', []):
for split_line in line.split('\n'):
script.command(split_line)
else:
script.comment('No commands given for this script.')
script.write()
def test_writeScript_2( self ):
"""Testing the writeScript function of the ScriptComposer class."""
testHeaderShell = "/usr/env/python"
testHeaderScheduler = OrderedDict()
testHeaderScheduler[ '-G' ] = 'pam'
testHeaderScheduler[ '-N' ] = 'fam'
testDetailsName = 'testName'
testDetailsType = 'batch'
testDetailsUser = os.environ['USER']
testDetailsGroup = self._other_group()
testDetailsOP = 7
testDetailsGP = 6
testDetailsWP = 0
testComposer = scriptcomposer.ScriptComposer()
testComposer.header.shell_path = testHeaderShell
testComposer.header.scheduler_macros = testHeaderScheduler
testComposer.body.name = testDetailsName
testComposer.body.script_type = testDetailsType
testComposer.body.user = testDetailsUser
testComposer.body.group = testDetailsGroup
testComposer.body.owner_perms = testDetailsOP
testComposer.body.group_perms = testDetailsGP
testComposer.body.world_perms = testDetailsWP
testDir = os.getcwd()
testComposer.write_script(testDir)
self.assertTrue( os.path.isfile( os.path.join( testDir,
testDetailsName + '.batch' ) ) )
testFile = open(os.path.join( testDir, testDetailsName+'.batch'), 'r' )
testLines = testFile.readlines()
testFile.close()
for i in range(0, len( testLines ) ):
testLines[i] = testLines[i].strip()
self.assertEqual( testLines[0], "#!/usr/env/python" )
self.assertEqual( testLines[1], "# -G pam" )
self.assertEqual( testLines[2], "# -N fam" )
self.assertEqual( testLines[3], "" )
self.assertEqual( len(testLines), 4 )
testStat = os.stat( os.path.join(testDir, testDetailsName + '.batch') )
expectedStat=stat.S_IFREG + stat.S_IRWXU + stat.S_IRGRP + stat.S_IWGRP
self.assertEqual( testStat.st_mode, expectedStat )
testUID = testStat.st_uid
testGID = testStat.st_gid
testUID = pwd.getpwuid( testUID ).pw_name
testGID = grp.getgrgid( testGID ).gr_name
self.assertEqual( testUID, testDetailsUser )
self.assertEqual( testGID, testDetailsGroup )
os.remove( os.path.join( testDir, testDetailsName + '.batch' ) )
def test_scriptComposer( self ):
"""Testing ScriptComposer class variable setting."""
# Testing valid uses.
# Testing initialization defaults.
testComposer = scriptcomposer.ScriptComposer()
self.assertIsInstance(testComposer.header,
scriptcomposer.ScriptHeader)
self.assertIsInstance(testComposer.body,
scriptcomposer.ScriptBody)
self.assertIsNone( testComposer.header.shell_path )
self.assertIsNone( testComposer.header.scheduler_macros )
self.assertEqual(testComposer.body.script_type, "bash")
self.assertEqual(testComposer.body.user, os.environ['USER'])
self.assertEqual(testComposer.body.group, os.environ['USER'])
self.assertEqual(testComposer.body.owner_perms, 7)
self.assertEqual(testComposer.body.group_perms, 5)
self.assertEqual(testComposer.body.world_perms, 0)
# Testing individual assignment
testHeaderShell = "/usr/env/python"
testHeaderScheduler = OrderedDict()
testHeaderScheduler[ '-G' ] = 'pam'
testHeaderScheduler[ '-N' ] = 'fam'
testComposer.newline()
testComposer.command(['taco', 'burrito', 'nachos'])
testDetailsName = 'testName'
testDetailsType = 'batch'
testDetailsUser = '******'
testDetailsGroup = 'groupies'
testDetailsOP = 1
testDetailsGP = 2
testDetailsWP = 3
testComposer.header.shell_path = testHeaderShell
testComposer.header.scheduler_macros = testHeaderScheduler
testComposer.body.name = testDetailsName
testComposer.body.script_type = testDetailsType
testComposer.body.user = testDetailsUser
testComposer.body.group = testDetailsGroup
testComposer.body.owner_perms = testDetailsOP
testComposer.body.group_perms = testDetailsGP
testComposer.body.world_perms = testDetailsWP
self.assertEqual( testComposer.header.shell_path, testHeaderShell )
self.assertEqual( testComposer.header.scheduler_macros,
testHeaderScheduler )
self.assertEqual(testComposer.body.name, testDetailsName)
self.assertEqual(testComposer.body.script_type, testDetailsType)
self.assertEqual(testComposer.body.user, testDetailsUser)
self.assertEqual(testComposer.body.group, testDetailsGroup)
self.assertEqual(testComposer.body.owner_perms, testDetailsOP)
self.assertEqual(testComposer.body.group_perms, testDetailsGP)
self.assertEqual(testComposer.body.world_perms, testDetailsWP)
# Testing reset.
testComposer.reset()
self.assertIsNone( testComposer.header.shell_path )
self.assertIsNone( testComposer.header.scheduler_macros )
self.assertEqual(testComposer.body.script_type, "bash")
self.assertEqual(testComposer.body.user, os.environ['USER'])
self.assertEqual(testComposer.body.group, os.environ['USER'])
self.assertEqual(testComposer.body.owner_perms, 7)
self.assertEqual(testComposer.body.group_perms, 5)
self.assertEqual(testComposer.body.world_perms, 0)
# Testing object assignment.
testHeaderObj = scriptcomposer.ScriptHeader(shell_path=testHeaderShell,
scheduler_macros=
testHeaderScheduler)
testDetailsObj = scriptcomposer.ScriptBody(name=testDetailsName,
script_type=
testDetailsType,
user=testDetailsUser,
group=testDetailsGroup,
owner_perms=
testDetailsOP,
group_perms=
testDetailsGP,
world_perms=
testDetailsWP)
testComposer.header = testHeaderObj
testComposer.body = testDetailsObj
self.assertEqual( testComposer.header.shell_path, testHeaderShell )
self.assertEqual( testComposer.header.scheduler_macros,
testHeaderScheduler )
self.assertEqual(testComposer.body.name, testDetailsName)
self.assertEqual(testComposer.body.script_type, testDetailsType)
self.assertEqual(testComposer.body.user, testDetailsUser)
self.assertEqual(testComposer.body.group, testDetailsGroup)
self.assertEqual(testComposer.body.owner_perms, testDetailsOP)
self.assertEqual(testComposer.body.group_perms, testDetailsGP)
self.assertEqual(testComposer.body.world_perms, testDetailsWP)
def test_scriptComposer(self):
"""Testing ScriptComposer class variable setting."""
# Testing valid uses.
# Testing initialization defaults.
composer = scriptcomposer.ScriptComposer()
self.assertIsInstance(composer.header, scriptcomposer.ScriptHeader)
self.assertIsInstance(composer.details, scriptcomposer.ScriptDetails)
self.assertEqual(composer.header.shell_path, '#!/bin/bash')
self.assertEqual(composer.header.scheduler_headers, [])
self.assertEqual(composer.details.group, utils.get_login())
self.assertEqual(composer.details.perms, oct(0o770))
# Testing individual assignment
test_header_shell = "/usr/env/python"
test_header_scheduler = OrderedDict()
test_header_scheduler['-G'] = 'pam'
test_header_scheduler['-N'] = 'fam'
composer.newline()
composer.command(['taco', 'burrito', 'nachos'])
test_details_path = 'testPath'
test_details_group = 'groupies'
test_details_perms = 0o543
composer.header.shell_path = test_header_shell
composer.header.scheduler_headers = test_header_scheduler
composer.details.path = test_details_path
composer.details.group = test_details_group
composer.details.perms = test_details_perms
self.assertEqual(composer.header.shell_path, test_header_shell)
self.assertEqual(composer.header.scheduler_headers,
test_header_scheduler)
self.assertEqual(composer.details.path, Path(test_details_path))
self.assertEqual(composer.details.group, test_details_group)
self.assertEqual(composer.details.perms, oct(test_details_perms))
composer = scriptcomposer.ScriptComposer()
self.assertEqual(composer.header.shell_path, '#!/bin/bash')
self.assertEqual(composer.header.scheduler_headers, [])
self.assertEqual(composer.details.group, utils.get_login())
self.assertEqual(composer.details.perms, oct(0o770))
# Testing object assignment.
header = scriptcomposer.ScriptHeader(
shell_path=test_header_shell,
scheduler_headers=test_header_scheduler)
testDetailsObj = scriptcomposer.ScriptDetails(path=test_details_path,
group=test_details_group,
perms=test_details_perms)
composer.header = header
composer.details = testDetailsObj
self.assertEqual(composer.header.shell_path, test_header_shell)
self.assertEqual(composer.header.scheduler_headers,
test_header_scheduler)
self.assertEqual(composer.details.path, Path(test_details_path))
self.assertEqual(composer.details.group, test_details_group)
self.assertEqual(composer.details.perms, oct(test_details_perms))
def _write_script(self, stype, path, config):
"""Write a build or run script or template. The formats for each are
mostly identical.
:param str stype: The type of script (run or build).
:param Path path: Path to the template file to write.
:param dict config: Configuration dictionary for the script file.
:return:
"""
script = scriptcomposer.ScriptComposer()
verbose = config.get('verbose', 'false').lower() == 'true'
if verbose:
script.comment('# Echoing all commands to log.')
script.command('set -v')
script.newline()
pav_lib_bash = self._pav_cfg.pav_root / 'bin' / 'pav-lib.bash'
# If we include this directly, it breaks build hashing.
script.comment('The first (and only) argument of the build script is '
'the test id.')
script.env_change({
'TEST_ID': '${1:-0}', # Default to test id 0 if one isn't given.
'PAV_CONFIG_FILE': self._pav_cfg['pav_cfg_file']
})
script.command('source {}'.format(pav_lib_bash))
if config.get('preamble', []):
script.newline()
script.comment('Preamble commands')
for cmd in config['preamble']:
script.command(cmd)
if stype == 'build' and not self.build_local:
script.comment('To be built in an allocation.')
modules = config.get('modules', [])
if modules:
script.newline()
script.comment(
'Perform module related changes to the environment.')
for module in config.get('modules', []):
script.module_change(module, self.var_man)
env = config.get('env', {})
if env:
script.newline()
script.comment("Making any environment changes needed.")
script.env_change(config.get('env', {}))
if verbose:
script.newline()
script.comment('List all the module modules for posterity')
script.command("module -t list")
script.newline()
script.comment('Output the environment for posterity')
script.command("declare -p")
if self.spack_enabled():
spack_commands = config.get('spack', {})
install_packages = spack_commands.get('install', [])
load_packages = spack_commands.get('load', [])
script.newline()
script.comment('Source spack setup script.')
script.command('source {}/share/spack/setup-env.sh'.format(
self._pav_cfg.get('spack_path')))
script.newline()
script.command('spack env deactivate &>/dev/null')
script.comment('Activate spack environment.')
script.command('spack env activate -d .')
script.command('if [ -z $SPACK_ENV ]; then exit 1; fi')
if install_packages:
script.newline()
script.comment('Install spack packages.')
for package in install_packages:
script.command(
'spack install -v --fail-fast {} || exit 1'.format(
package))
if load_packages:
script.newline()
script.comment('Load spack packages.')
for package in load_packages:
script.command('spack load {} || exit 1'.format(package))
script.newline()
cmds = config.get('cmds', [])
if cmds:
script.comment("Perform the sequence of test commands.")
for line in config.get('cmds', []):
for split_line in line.split('\n'):
script.command(split_line)
else:
script.comment('No commands given for this script.')
with PermissionsManager(path, self.group, self.umask):
script.write(path)
