def run(self): input_fas = self.input_files_local[0][0:2] output_fas = self.output_files_local() output_sam_file = os.path.join( self.output_dir_local, self.additional_attributes["output_sam_file"]) self.additional_output_files_hidden.append(output_sam_file) genome_dir = fetch_reference(self.additional_files["gsnap_genome"], self.ref_dir_local, allow_s3mi=True, auto_untar=True) gsnap_base_dir = os.path.dirname(genome_dir) gsnap_index_name = os.path.basename(genome_dir) # Run Gsnap gsnap_params = [ '-A', 'sam', '--batch=0', '--use-shared-memory=0', '--gmap-mode=all', '--npaths=1', '--ordered', '-t', 32, '--max-mismatches=40', '-D', gsnap_base_dir, '-d', gsnap_index_name, '-o', output_sam_file ] + input_fas command.execute( command_patterns.SingleCommand(cmd='gsnapl', args=gsnap_params)) log.write("Finished GSNAP alignment.") # Extract out unmapped files from sam if len(input_fas) == 2: convert.generate_unmapped_pairs_from_sam(output_sam_file, output_fas) else: convert.generate_unmapped_singles_from_sam(output_sam_file, output_fas[0])
def run(self): input_fas = self.input_fas() output_fas = self.output_files_local() genome_dir = fetch_reference( self.additional_files["bowtie2_genome"], self.ref_dir_local, allow_s3mi=True, auto_untar=True) output_sam_file = os.path.join( self.output_dir_local, self.additional_attributes["output_sam_file"]) self.additional_output_files_hidden.append(output_sam_file) # The file structure looks like # "bowtie2_genome/GRCh38.primary_assembly.genome.3.bt2" genome_basename = command.glob(f"{genome_dir}/*.bt2*", max_results=1)[0] # remove two extensions: ex: hg38_phiX_rRNA_mito_ERCC.3.bt2 -> hg38_phiX_rRNA_mito_ERCC genome_basename = os.path.splitext(os.path.splitext(genome_basename)[0])[0] bowtie2_params = [ '-q', '-x', genome_basename, '-f', '--very-sensitive-local', '-S', output_sam_file ] # --seed cannot be used with -p multithreading # We have observed the lack of multithreading resulting in # severe performance degradation in some cases. So for the # time being multithreading is being chosen over determinism. # To seed bowtie2 do something similar to: # bowtie2_params.extend(['--seed', '4']) bowtie2_params.extend(['-p', str(multiprocessing.cpu_count())]) if len(input_fas) == 2: bowtie2_params.extend(['-1', input_fas[0], '-2', input_fas[1]]) else: bowtie2_params.extend(['-U', input_fas[0]]) # Example: # bowtie2 -q -x /mnt/idseq/ref/bowtie2_genome/hg38_phiX_rRNA_mito_ERCC -f \ # --very-sensitive-local -S /mnt/idseq/results/589/bowtie2_human.sam \ # -p 32 \ # -1 /mnt/idseq/results/589/unmapped_human_1.fa -2 /mnt/idseq/results/589/unmapped_human_2.fa command.execute( command_patterns.SingleCommand( cmd='bowtie2', args=bowtie2_params ) ) log.write("Finished Bowtie alignment.") if len(input_fas) == 2: convert.generate_unmapped_pairs_from_sam(output_sam_file, output_fas) else: convert.generate_unmapped_singles_from_sam(output_sam_file, output_fas[0])
def run(self): input_fas = self.input_files_local[0][0:2] output_fas = self.output_files_local() genome_dir = fetch_reference( self.additional_files["bowtie2_genome"], self.ref_dir_local, allow_s3mi=True, auto_untar=True) output_sam_file = os.path.join( self.output_dir_local, self.additional_attributes["output_sam_file"]) self.additional_output_files_hidden.append(output_sam_file) # The file structure looks like # "bowtie2_genome/GRCh38.primary_assembly.genome.3.bt2" genome_basename = command.glob(f"{genome_dir}/*.bt2*", max_results=1)[0] # remove two extensions: ex: hg38_phiX_rRNA_mito_ERCC.3.bt2 -> hg38_phiX_rRNA_mito_ERCC genome_basename = os.path.splitext(os.path.splitext(genome_basename)[0])[0] bowtie2_params = [ '-q', '-x', genome_basename, '-f', '--very-sensitive-local', '-S', output_sam_file ] seed = self.additional_attributes.get("random_seed") if seed: bowtie2_params.extend(['--seed', str(seed)]) else: # Seed option won't work with -p threading option. bowtie2_params.extend(['-p', str(multiprocessing.cpu_count())]) if len(input_fas) == 2: bowtie2_params.extend(['-1', input_fas[0], '-2', input_fas[1]]) else: bowtie2_params.extend(['-U', input_fas[0]]) # Example: # bowtie2 -q -x /mnt/idseq/ref/bowtie2_genome/hg38_phiX_rRNA_mito_ERCC -f \ # --very-sensitive-local -S /mnt/idseq/results/589/bowtie2_human.sam \ # -p 32 \ # -1 /mnt/idseq/results/589/unmapped_human_1.fa -2 /mnt/idseq/results/589/unmapped_human_2.fa command.execute( command_patterns.SingleCommand( cmd='bowtie2', args=bowtie2_params ) ) log.write("Finished Bowtie alignment.") if len(input_fas) == 2: convert.generate_unmapped_pairs_from_sam(output_sam_file, output_fas) else: convert.generate_unmapped_singles_from_sam(output_sam_file, output_fas[0])
def run(self): input_fas = self.input_fas() output_fas = self.output_files_local() genome_dir = fetch_reference(self.additional_files["bowtie2_genome"], self.ref_dir_local, allow_s3mi=True, auto_untar=True) output_sam_file = os.path.join( self.output_dir_local, self.additional_attributes["output_sam_file"]) self.additional_output_files_hidden.append(output_sam_file) # The file structure looks like # "bowtie2_genome/GRCh38.primary_assembly.genome.3.bt2" genome_basename = command.glob(f"{genome_dir}/*.bt2*", max_results=1)[0] # remove two extensions: ex: hg38_phiX_rRNA_mito_ERCC.3.bt2 -> hg38_phiX_rRNA_mito_ERCC genome_basename = os.path.splitext( os.path.splitext(genome_basename)[0])[0] bowtie2_params = [ '-q', '-x', genome_basename, '-f', '--very-sensitive-local', '-S', output_sam_file ] # FIXME: https://jira.czi.team/browse/IDSEQ-2738 # We want to move towards a general randomness solution in which # all randomness is seeded based on the content of the original input. # This is currently introducing non-determinism and hard coding # an arbitrary seed here shouldn't impact correctness. bowtie2_params.extend( ['--seed', '4']) # chosen by fair dice role, guaranteed to be random if len(input_fas) == 2: bowtie2_params.extend(['-1', input_fas[0], '-2', input_fas[1]]) else: bowtie2_params.extend(['-U', input_fas[0]]) # Example: # bowtie2 -q -x /mnt/idseq/ref/bowtie2_genome/hg38_phiX_rRNA_mito_ERCC -f \ # --very-sensitive-local -S /mnt/idseq/results/589/bowtie2_human.sam \ # -p 32 \ # -1 /mnt/idseq/results/589/unmapped_human_1.fa -2 /mnt/idseq/results/589/unmapped_human_2.fa command.execute( command_patterns.SingleCommand(cmd='bowtie2', args=bowtie2_params)) log.write("Finished Bowtie alignment.") if len(input_fas) == 2: convert.generate_unmapped_pairs_from_sam(output_sam_file, output_fas) else: convert.generate_unmapped_singles_from_sam(output_sam_file, output_fas[0])
def run(self): input_fas = self.input_files_local[0][0:2] output_fas = self.output_files_local() genome_dir = fetch_from_s3(self.additional_files["bowtie2_genome"], self.ref_dir_local, allow_s3mi=True, auto_untar=True) output_sam_file = os.path.join( self.output_dir_local, self.additional_attributes["output_sam_file"]) self.additional_files_to_upload.append(output_sam_file) # The file structure looks like # "bowtie2_genome/GRCh38.primary_assembly.genome.3.bt2" # The code below will handle up to "bowtie2_genome/GRCh38.primary_assembly. # genome.99.bt2" but not 100. cmd = "ls {genome_dir}/*.bt2*".format(genome_dir=genome_dir) local_genome_dir_ls = command.execute_with_output(cmd) genome_basename = local_genome_dir_ls.split("\n")[0][:-6] if genome_basename[-1] == '.': genome_basename = genome_basename[:-1] bowtie2_params = [ 'bowtie2', '-q', '-x', genome_basename, '-f', '--very-sensitive-local', '-S', output_sam_file ] seed = self.additional_attributes.get("random_seed") if seed: bowtie2_params.extend(['--seed', str(seed)]) else: # Seed option won't work with -p threading option. bowtie2_params.extend(['-p', str(multiprocessing.cpu_count())]) if len(input_fas) == 2: bowtie2_params.extend(['-1', input_fas[0], '-2', input_fas[1]]) else: bowtie2_params.extend(['-U', input_fas[0]]) command.execute(" ".join(bowtie2_params)) log.write("Finished Bowtie alignment.") if len(input_fas) == 2: convert.generate_unmapped_pairs_from_sam(output_sam_file, output_fas) else: convert.generate_unmapped_singles_from_sam(output_sam_file, output_fas[0])
def run(self): input_fas = self.input_fas() output_fas = self.output_files_local() output_sam_file = os.path.join(self.output_dir_local, self.additional_attributes["output_sam_file"]) self.additional_output_files_hidden.append(output_sam_file) genome_dir = fetch_reference(self.additional_files["gsnap_genome"], self.ref_dir_local, allow_s3mi=True, auto_untar=True) gsnap_base_dir = os.path.dirname(genome_dir) gsnap_index_name = os.path.basename(genome_dir) # Hack to determine gsnap vs gsnapl error_message = subprocess.run( ['gsnapl', '-D', gsnap_base_dir, '-d', gsnap_index_name], input='>'.encode('utf-8'), stderr=subprocess.PIPE, stdout=subprocess.PIPE ).stderr gsnap_exe = "gsnap" if 'please run gsnap instead' in error_message.decode('utf-8') else "gsnapl" # Run Gsnap gsnap_params = [ '-A', 'sam', '--batch=0', '--use-shared-memory=0', '--gmap-mode=all', '--npaths=1', '--ordered', '-t', 32, '--max-mismatches=40', '-D', gsnap_base_dir, '-d', gsnap_index_name, '-o', output_sam_file ] + input_fas command.execute( command_patterns.SingleCommand( cmd=gsnap_exe, args=gsnap_params ) ) log.write("Finished GSNAP alignment.") # Extract out unmapped files from sam if len(input_fas) == 2: convert.generate_unmapped_pairs_from_sam( output_sam_file, output_fas) else: convert.generate_unmapped_singles_from_sam( output_sam_file, output_fas[0])