def setUp(self):
self.fs = RemoteFileSystem(working_ssh_host)
self.root = '/tmp/luigi-remote-test'
self.directory = self.root + '/dir'
self.filepath = self.directory + '/file'
self.target = RemoteTarget(
self.filepath,
working_ssh_host,
)
self.fs.remote_context.check_output(['rm', '-rf', self.root])
self.addCleanup(self.fs.remote_context.check_output, ['rm', '-rf', self.root])
class TestRemoteFilesystem(unittest.TestCase):
def setUp(self):
self.fs = RemoteFileSystem(working_ssh_host)
self.root = '/tmp/luigi-remote-test'
self.directory = self.root + '/dir'
self.filepath = self.directory + '/file'
self.target = RemoteTarget(
self.filepath,
working_ssh_host,
)
self.fs.remote_context.check_output(['rm', '-rf', self.root])
self.addCleanup(self.fs.remote_context.check_output, ['rm', '-rf', self.root])
def test_mkdir(self):
self.assertFalse(self.fs.isdir(self.directory))
self.assertRaises(MissingParentDirectory, self.fs.mkdir, self.directory, parents=False)
self.fs.mkdir(self.directory)
self.assertTrue(self.fs.isdir(self.directory))
# Shouldn't throw
self.fs.mkdir(self.directory)
self.assertRaises(FileAlreadyExists, self.fs.mkdir, self.directory, raise_if_exists=True)
def test_list(self):
with self.target.open('w'):
pass
self.assertEqual([self.target.path], list(self.fs.listdir(self.directory)))
class TestRemoteFilesystem(unittest.TestCase):
def setUp(self):
self.fs = RemoteFileSystem(working_ssh_host)
self.root = '/tmp/luigi-remote-test'
self.directory = self.root + '/dir'
self.filepath = self.directory + '/file'
self.target = RemoteTarget(
self.filepath,
working_ssh_host,
)
self.fs.remote_context.check_output(['rm', '-rf', self.root])
self.addCleanup(self.fs.remote_context.check_output, ['rm', '-rf', self.root])
def test_mkdir(self):
self.assertFalse(self.fs.isdir(self.directory))
self.assertRaises(MissingParentDirectory, self.fs.mkdir, self.directory, parents=False)
self.fs.mkdir(self.directory)
self.assertTrue(self.fs.isdir(self.directory))
# Shouldn't throw
self.fs.mkdir(self.directory)
self.assertRaises(FileAlreadyExists, self.fs.mkdir, self.directory, raise_if_exists=True)
def test_list(self):
with self.target.open('w'):
pass
self.assertEquals([self.target.path], list(self.fs.listdir(self.directory)))
def setUp(self):
self.fs = RemoteFileSystem(working_ssh_host)
self.root = '/tmp/luigi-remote-test'
self.directory = self.root + '/dir'
self.filepath = self.directory + '/file'
self.target = RemoteTarget(
self.filepath,
working_ssh_host,
)
self.fs.remote_context.check_output(['rm', '-rf', self.root])
self.addCleanup(self.fs.remote_context.check_output, ['rm', '-rf', self.root])
def run(self):
# copy input to remote host
# Remote targets need to be copied first to local system
inp = self.input()
if isinstance(inp, RemoteTarget):
inp.get(inp.path)
remote_fs = RemoteFileSystem(self.ssh_host)
remote_path = os.path.join(self.remote_dir, self.input().path)
remote_fs.put(self.input().path, remote_path)
infile = self.input().path
outfile = os.path.basename(self.output().path)
remote = RemoteContext(self.ssh_host)
remote.check_output(
['cd {};'.format(self.remote_dir), self.exe, infile, outfile])
# clean up intermediate files
if isinstance(inp, RemoteTarget):
os.remove(infile)
remote_fs.remove(remote_path)
if self.remove_input:
self.input().remove()
def run(self):
# copy input to remote host
# Remote targets need to be copied first to local system
inp = self.input()
if isinstance(inp, RemoteTarget):
inp.get(inp.path)
remote_fs = RemoteFileSystem(self.ssh_host)
remote_path = os.path.join(self.remote_dir, self.input().path)
remote_fs.put(self.input().path, remote_path)
infile = self.input().path
outfile = os.path.basename(self.output().path)
# create slurm batch script
with open('slurm_job.sh', 'w') as f:
f.write('#!/bin/bash\n')
f.write('#SBATCH --job-name=luigi-task\n')
f.write('#SBATCH --time={}\n'.format(self.slurm_time))
f.write('#SBATCH --partition={}\n'.format(self.slurm_partition))
f.write('#SBATCH --ntasks=1\n')
f.write('#SBATCH --account=project_2002078\n')
f.write('\n')
f.write('srun {} {} {}\n'.format(self.exe, infile, outfile))
f.write('\n')
# copy batch script to remote host
remote_path = os.path.join(self.remote_dir, 'slurm_job.sh')
remote_fs.put('slurm_job.sh', remote_path)
os.remove('slurm_job.sh')
# Submit batch job and wait for its completion
remote = RemoteContext(self.ssh_host)
slurm_out = remote.check_output(
['cd {};'.format(self.remote_dir), 'sbatch', 'slurm_job.sh'])
job_id = int(slurm_out.split()[-1])
sacct_cmd = 'sacct -n -X -j {}'.format(job_id)
# Wait for job to complete
sleep(20)
while True:
sacct_out = remote.check_output([sacct_cmd]).decode('utf-8')
if 'COMPLETED' in sacct_out:
print("DEB: completed")
break
elif 'FAILED' in sacct_out:
self.output().remove()
raise RuntimeError('Task {} failed'.format(
self.__class__.__name__))
else:
# Do not poll sacct too frequently
sleep(60)
def run(self):
remote = RemoteFileSystem(SSH_HOST)
remote.put(config.local_storage + '/' + parameters.data['input'],
parameters.data['remote_storage'] + 'rawreads.fasta')
def run(self):
remote = RemoteFileSystem(SSH_HOST)
remote.put(parameters.data['barcode'],
parameters.data['remote_storage'] + 'barcodes.fasta')
