def setUp(self): self.fs = RemoteFileSystem(working_ssh_host) self.root = '/tmp/luigi-remote-test' self.directory = self.root + '/dir' self.filepath = self.directory + '/file' self.target = RemoteTarget( self.filepath, working_ssh_host, ) self.fs.remote_context.check_output(['rm', '-rf', self.root]) self.addCleanup(self.fs.remote_context.check_output, ['rm', '-rf', self.root])
class TestRemoteFilesystem(unittest.TestCase): def setUp(self): self.fs = RemoteFileSystem(working_ssh_host) self.root = '/tmp/luigi-remote-test' self.directory = self.root + '/dir' self.filepath = self.directory + '/file' self.target = RemoteTarget( self.filepath, working_ssh_host, ) self.fs.remote_context.check_output(['rm', '-rf', self.root]) self.addCleanup(self.fs.remote_context.check_output, ['rm', '-rf', self.root]) def test_mkdir(self): self.assertFalse(self.fs.isdir(self.directory)) self.assertRaises(MissingParentDirectory, self.fs.mkdir, self.directory, parents=False) self.fs.mkdir(self.directory) self.assertTrue(self.fs.isdir(self.directory)) # Shouldn't throw self.fs.mkdir(self.directory) self.assertRaises(FileAlreadyExists, self.fs.mkdir, self.directory, raise_if_exists=True) def test_list(self): with self.target.open('w'): pass self.assertEqual([self.target.path], list(self.fs.listdir(self.directory)))
class TestRemoteFilesystem(unittest.TestCase): def setUp(self): self.fs = RemoteFileSystem(working_ssh_host) self.root = '/tmp/luigi-remote-test' self.directory = self.root + '/dir' self.filepath = self.directory + '/file' self.target = RemoteTarget( self.filepath, working_ssh_host, ) self.fs.remote_context.check_output(['rm', '-rf', self.root]) self.addCleanup(self.fs.remote_context.check_output, ['rm', '-rf', self.root]) def test_mkdir(self): self.assertFalse(self.fs.isdir(self.directory)) self.assertRaises(MissingParentDirectory, self.fs.mkdir, self.directory, parents=False) self.fs.mkdir(self.directory) self.assertTrue(self.fs.isdir(self.directory)) # Shouldn't throw self.fs.mkdir(self.directory) self.assertRaises(FileAlreadyExists, self.fs.mkdir, self.directory, raise_if_exists=True) def test_list(self): with self.target.open('w'): pass self.assertEquals([self.target.path], list(self.fs.listdir(self.directory)))
def setUp(self): self.fs = RemoteFileSystem(working_ssh_host) self.root = '/tmp/luigi-remote-test' self.directory = self.root + '/dir' self.filepath = self.directory + '/file' self.target = RemoteTarget( self.filepath, working_ssh_host, ) self.fs.remote_context.check_output(['rm', '-rf', self.root]) self.addCleanup(self.fs.remote_context.check_output, ['rm', '-rf', self.root])
def run(self): # copy input to remote host # Remote targets need to be copied first to local system inp = self.input() if isinstance(inp, RemoteTarget): inp.get(inp.path) remote_fs = RemoteFileSystem(self.ssh_host) remote_path = os.path.join(self.remote_dir, self.input().path) remote_fs.put(self.input().path, remote_path) infile = self.input().path outfile = os.path.basename(self.output().path) remote = RemoteContext(self.ssh_host) remote.check_output( ['cd {};'.format(self.remote_dir), self.exe, infile, outfile]) # clean up intermediate files if isinstance(inp, RemoteTarget): os.remove(infile) remote_fs.remove(remote_path) if self.remove_input: self.input().remove()
def run(self): # copy input to remote host # Remote targets need to be copied first to local system inp = self.input() if isinstance(inp, RemoteTarget): inp.get(inp.path) remote_fs = RemoteFileSystem(self.ssh_host) remote_path = os.path.join(self.remote_dir, self.input().path) remote_fs.put(self.input().path, remote_path) infile = self.input().path outfile = os.path.basename(self.output().path) # create slurm batch script with open('slurm_job.sh', 'w') as f: f.write('#!/bin/bash\n') f.write('#SBATCH --job-name=luigi-task\n') f.write('#SBATCH --time={}\n'.format(self.slurm_time)) f.write('#SBATCH --partition={}\n'.format(self.slurm_partition)) f.write('#SBATCH --ntasks=1\n') f.write('#SBATCH --account=project_2002078\n') f.write('\n') f.write('srun {} {} {}\n'.format(self.exe, infile, outfile)) f.write('\n') # copy batch script to remote host remote_path = os.path.join(self.remote_dir, 'slurm_job.sh') remote_fs.put('slurm_job.sh', remote_path) os.remove('slurm_job.sh') # Submit batch job and wait for its completion remote = RemoteContext(self.ssh_host) slurm_out = remote.check_output( ['cd {};'.format(self.remote_dir), 'sbatch', 'slurm_job.sh']) job_id = int(slurm_out.split()[-1]) sacct_cmd = 'sacct -n -X -j {}'.format(job_id) # Wait for job to complete sleep(20) while True: sacct_out = remote.check_output([sacct_cmd]).decode('utf-8') if 'COMPLETED' in sacct_out: print("DEB: completed") break elif 'FAILED' in sacct_out: self.output().remove() raise RuntimeError('Task {} failed'.format( self.__class__.__name__)) else: # Do not poll sacct too frequently sleep(60)
def run(self): remote = RemoteFileSystem(SSH_HOST) remote.put(config.local_storage + '/' + parameters.data['input'], parameters.data['remote_storage'] + 'rawreads.fasta')
def run(self): remote = RemoteFileSystem(SSH_HOST) remote.put(parameters.data['barcode'], parameters.data['remote_storage'] + 'barcodes.fasta')