def test_recursive(self):
builder1 = cmd_builder.CommandBuilder()
builder2 = cmd_builder.CommandBuilder()
builder1.append('a')
builder1.append(builder2)
builder1.append('c')
builder2.append('b')
self.assertEqual(str(builder1), 'a;b;c')
self.assertEqual(str(builder2), 'b')
def write_singularity_hook(arch_dir, image_name, mounts,
script_name,
extra_flags='',
verbose=False, use_nvidia_docker=False):
singularity_hook_file = os.path.join(arch_dir, script_name)
builder = cmd_builder.CommandBuilder()
builder.append('#!/bin/bash')
mnt_cmd = ' '.join(['--bind %s:%s' % (mnt.sync_dir, mnt.mount_point)
for mnt in mounts if mnt.writeable])
tmp_dir = tempfile.mkdtemp()
def create_bind_flag(mnt):
parent_directory = pathlib.Path(mnt.mount_point).parent
return '--bind %s:%s' % (tmp_dir, parent_directory)
mnt_cmd += ' ' + ' '.join([create_bind_flag(mnt)
for mnt in mounts if not mnt.writeable])
mnt_cmd += ' --bind $(pwd):/'+DAR_PAYLOAD_MOUNT
singularity_cmd = ('mkdir {tmp_dir}; singularity exec {extra_flags} {gpu_opt} {mount_cmds} {img} /bin/bash -c "cd /{dar_payload}; ./run.sh $*"'.format(
tmp_dir=tmp_dir,
gpu_opt='--nv' if use_nvidia_docker else '',
img=image_name,
extra_flags=extra_flags,
mount_cmds=mnt_cmd,
dar_payload=DAR_PAYLOAD_MOUNT
))
if verbose:
builder.echo('Singularity command:' + singularity_cmd)
builder.append(singularity_cmd)
with open(singularity_hook_file, 'w') as f:
f.write(builder.dump_script())
os.chmod(singularity_hook_file, 0o777)
def write_docker_hook(
arch_dir, image_name, mounts, script_name,
extra_flags='',
verbose=False, use_nvidia_docker=False):
docker_hook_file = os.path.join(arch_dir, script_name)
builder = cmd_builder.CommandBuilder()
builder.append('#!/bin/bash')
#if verbose:
# builder.echo('All script arguments:')
# builder.echo('$@')
mnt_cmd = ''.join([' -v %s:%s' % (mnt.sync_dir, mnt.mount_point)
for mnt in mounts if mnt.writeable])
# mount the script into the docker image
mnt_cmd += ' -v $(pwd):/'+DAR_PAYLOAD_MOUNT
docker_cmd = ('docker run {gpu_opt} {mount_cmds} {extra_flags} -t {img} /bin/bash -c "cd /{dar_payload};./run.sh $*"'.format(
gpu_opt='--gpus all' if use_nvidia_docker else '',
img=image_name,
extra_flags=extra_flags,
mount_cmds=mnt_cmd,
dar_payload=DAR_PAYLOAD_MOUNT
))
if verbose:
builder.echo('Docker command:' + docker_cmd)
builder.append(docker_cmd)
with open(docker_hook_file, 'w') as f:
f.write(builder.dump_script())
os.chmod(docker_hook_file, 0o777)
def write_run_script(arch_dir, mounts, payload_script, verbose=False):
runfile = os.path.join(arch_dir, 'run.sh')
builder = cmd_builder.CommandBuilder()
builder.append('#!/bin/bash')
if verbose:
builder.echo('Running Doodad Archive [DAR] $1')
builder.echo('CLI Args: $*')
builder.echo('DAR build information:')
builder.append('cat', './METADATA')
for mount in mounts:
if verbose:
builder.append('echo', 'Mounting %s' % mount)
builder.append(mount.dar_extract_command())
if mount.pythonpath:
builder.append('export PYTHONPATH=$PYTHONPATH:%s' %
mount.mount_point)
if verbose:
builder.append('echo', BEGIN_HEADER)
builder.append(payload_script + ' $*')
with open(runfile, 'w') as f:
f.write(builder.dump_script())
os.chmod(runfile, 0o777)
def test_args(self):
builder = cmd_builder.CommandBuilder()
builder.append('cat', '12', '34')
self.assertEqual(str(builder), 'cat 12 34')
def test_dump(self):
builder = cmd_builder.CommandBuilder()
builder.append('a')
builder.append('b')
builder.append('c')
self.assertEqual(builder.dump_script(), 'a\nb\nc')
def create_slurm_script(self, script):
cmd_list = cmd_builder.CommandBuilder()
cmd_list.append('module load gcc openmpi')
cmd_list.append('ht_helper.sh -m "python/3.5" -t {}'.format(
self.slurm_task_file_path))
super().create_slurm_script(cmd_list.to_string())