def run(self): # copy input to remote host # Remote targets need to be copied first to local system inp = self.input() if isinstance(inp, RemoteTarget): inp.get(inp.path) remote_fs = RemoteFileSystem(self.ssh_host) remote_path = os.path.join(self.remote_dir, self.input().path) remote_fs.put(self.input().path, remote_path) infile = self.input().path outfile = os.path.basename(self.output().path) remote = RemoteContext(self.ssh_host) remote.check_output( ['cd {};'.format(self.remote_dir), self.exe, infile, outfile]) # clean up intermediate files if isinstance(inp, RemoteTarget): os.remove(infile) remote_fs.remove(remote_path) if self.remove_input: self.input().remove()
def run(self): # copy input to remote host # Remote targets need to be copied first to local system inp = self.input() if isinstance(inp, RemoteTarget): inp.get(inp.path) remote_fs = RemoteFileSystem(self.ssh_host) remote_path = os.path.join(self.remote_dir, self.input().path) remote_fs.put(self.input().path, remote_path) infile = self.input().path outfile = os.path.basename(self.output().path) # create slurm batch script with open('slurm_job.sh', 'w') as f: f.write('#!/bin/bash\n') f.write('#SBATCH --job-name=luigi-task\n') f.write('#SBATCH --time={}\n'.format(self.slurm_time)) f.write('#SBATCH --partition={}\n'.format(self.slurm_partition)) f.write('#SBATCH --ntasks=1\n') f.write('#SBATCH --account=project_2002078\n') f.write('\n') f.write('srun {} {} {}\n'.format(self.exe, infile, outfile)) f.write('\n') # copy batch script to remote host remote_path = os.path.join(self.remote_dir, 'slurm_job.sh') remote_fs.put('slurm_job.sh', remote_path) os.remove('slurm_job.sh') # Submit batch job and wait for its completion remote = RemoteContext(self.ssh_host) slurm_out = remote.check_output( ['cd {};'.format(self.remote_dir), 'sbatch', 'slurm_job.sh']) job_id = int(slurm_out.split()[-1]) sacct_cmd = 'sacct -n -X -j {}'.format(job_id) # Wait for job to complete sleep(20) while True: sacct_out = remote.check_output([sacct_cmd]).decode('utf-8') if 'COMPLETED' in sacct_out: print("DEB: completed") break elif 'FAILED' in sacct_out: self.output().remove() raise RuntimeError('Task {} failed'.format( self.__class__.__name__)) else: # Do not poll sacct too frequently sleep(60)
def run(self): remote = RemoteFileSystem(SSH_HOST) remote.put(config.local_storage + '/' + parameters.data['input'], parameters.data['remote_storage'] + 'rawreads.fasta')
def run(self): remote = RemoteFileSystem(SSH_HOST) remote.put(parameters.data['barcode'], parameters.data['remote_storage'] + 'barcodes.fasta')