def test_check_and_report_dependencies(self): """test check_and_report_dependencies""" tmpfile = "tmp.check_and_report_dependencies.out" if os.path.exists(tmpfile): os.unlink(tmpfile) dependencies.check_and_report_dependencies(outfile=tmpfile) # We don't inow what the versions etc will be, so just # check file got written self.assertTrue(os.path.exists(tmpfile)) os.unlink(tmpfile) os.environ["MINOS_BWA"] = "oops_this_is_wrong" with self.assertRaises(Exception): dependencies.check_and_report_dependencies(outfile=tmpfile) del os.environ["MINOS_BWA"] self.assertTrue(os.path.exists(tmpfile)) os.unlink(tmpfile)
def test_check_and_report_dependencies(self): '''test check_and_report_dependencies''' tmpfile = 'tmp.check_and_report_dependencies.out' if os.path.exists(tmpfile): os.unlink(tmpfile) dependencies.check_and_report_dependencies(outfile=tmpfile) # We don't inow what the versions etc will be, so just # check file got written self.assertTrue(os.path.exists(tmpfile)) os.unlink(tmpfile) os.environ['MINOS_BWA'] = 'oops_this_is_wrong' with self.assertRaises(dependencies.Error): dependencies.check_and_report_dependencies(outfile=tmpfile) del os.environ['MINOS_BWA'] self.assertTrue(os.path.exists(tmpfile)) os.unlink(tmpfile)
def run(options): dependencies.check_and_report_dependencies("-")
def run(self): self._make_output_dir() fh = logging.FileHandler(self.log_file, mode='w') log = logging.getLogger() formatter = logging.Formatter( '[minos %(asctime)s %(levelname)s] %(message)s', datefmt='%d-%m-%Y %H:%M:%S') fh.setFormatter(formatter) log.addHandler(fh) dependencies.check_and_report_dependencies(programs=['nextflow']) self._prepare_nextflow_input_files() original_dir = os.getcwd() os.chdir(self.output_dir) nextflow_script = 'nextflow.run.nf' MultiSamplePipeline._write_nextflow_script(nextflow_script) logging.info('Prepared nextflow files. cd ' + self.output_dir) nextflow = dependencies.find_binary('nextflow') nextflow_command = [ nextflow, 'run', '-work-dir', self.nextflow_work_dir, '-with-dag', 'nextflow.out.dag.pdf', '-with-trace', 'nextflow.out.trace.txt', ] if self.nextflow_config_file is not None: nextflow_command.extend(['-c', self.nextflow_config_file]) nextflow_command += [ nextflow_script, '--ref_fasta', self.ref_fasta, '--data_in_tsv', self.nextflow_input_tsv, '--max_alleles_per_cluster', str(self.max_alleles_per_cluster), '--min_large_ref_length', str(self.min_large_ref_length), '--final_outdir', self.output_dir, '--gramtools_max_read_length', str(self.gramtools_max_read_length), '--cluster_small_vars_ram', str(self.nf_ram_cluster_small_vars), '--gramtools_build_small_vars_ram', str(self.nf_ram_gramtools_build_small), '--gramtools_kmer_size', str(self.gramtools_kmer_size), '--gramtools_build_threads', str(self.gramtools_build_threads), '--minos_small_vars_ram', str(self.nf_ram_minos_small_vars), '--merge_small_vars_ram', str(self.nf_ram_merge_small_vars), ] if self.testing: nextflow_command.append('--testing') if self.use_unmapped_reads: nextflow_command.append('--use_unmapped_reads') if self.variants_per_split is not None: nextflow_command.append('--variants_per_split ' + str(self.variants_per_split)) if self.alleles_per_split is not None: nextflow_command.append('--alleles_per_split ' + str(self.alleles_per_split)) elif self.total_splits is not None: nextflow_command.append('--total_splits ' + str(self.total_splits)) nextflow_command = ' '.join(nextflow_command) if self.no_run: print( 'Prepared nextflow pipeline. --no_run used, so not running. The nextflow command to run is:' ) print(nextflow_command) return else: logging.info('Start running nextflow: ' + nextflow_command) syscall_process = utils.syscall(nextflow_command) logging.info( 'Finish running nextflow. Writing nextflow stdout/stderr to files' ) with open('nextflow.stdout', 'w') as f: print(syscall_process.stdout.rstrip(), file=f) with open('nextflow.stderr', 'w') as f: print(syscall_process.stderr.rstrip(), file=f) logging.info('cd ' + original_dir) if self.clean: logging.info('Delete nextflow work directory ' + self.nextflow_work_dir) shutil.rmtree(self.nextflow_work_dir) logging.info('Delete .nextflow directory') shutil.rmtree('.nextflow') logging.info('Rename .nextflow.log -> nextflow.log') os.rename('.nextflow.log', 'nextflow.log') os.chdir(original_dir)
def run(self): if os.path.exists(self.outdir) and self.overwrite_outdir: shutil.rmtree(self.outdir) try: os.mkdir(self.outdir) except: raise Error('Error making output directory ' + self.outdir) fh = logging.FileHandler(self.log_file, mode='w') log = logging.getLogger() formatter = logging.Formatter( '[minos %(asctime)s %(levelname)s] %(message)s', datefmt='%d-%m-%Y %H:%M:%S') fh.setFormatter(formatter) log.addHandler(fh) logging.info('Command run: ' + ' '.join(sys.argv)) dependencies.check_and_report_dependencies(programs=['gramtools']) logging.info('Dependencies look OK') if self.read_error_rate is None or self.max_read_length is None: logging.info( 'One or both of read_error_rate and max_read_length not known. Estimate from first 10,000 reads...' ) estimated_read_length, estimated_read_error_rate = utils.estimate_max_read_length_and_read_error_rate_from_qual_scores( self.reads_files[0]) logging.info('Estimated max_read_length=' + str(estimated_read_length) + ' and read_error_rate=' + str(estimated_read_error_rate)) self.read_error_rate = estimated_read_error_rate if self.read_error_rate is None else self.read_error_rate self.max_read_length = estimated_read_length if self.max_read_length is None else self.max_read_length logging.info('Using max_read_length=' + str(self.max_read_length) + ' and read_error_rate=' + str(self.read_error_rate)) if self.user_supplied_gramtools_build_dir: logging.info( 'User supplied gramtools build dir. Assuming VCF already clustered, so skipping clustering' ) assert len(self.vcf_files) == 1 self.clustered_vcf = self.vcf_files[0] else: logging.info( 'Clustering VCF file(s), to make one VCF input file for gramtools' ) clusterer = vcf_clusterer.VcfClusterer( self.vcf_files, self.ref_fasta, self.clustered_vcf, max_distance_between_variants=1, max_alleles_per_cluster=self.max_alleles_per_cluster, ) clusterer.run() logging.info('Finished clustering VCF file(s)') if not vcf_file_read.vcf_file_has_at_least_one_record( self.clustered_vcf): error_message = 'No VCF records. Cannot continue. Please check that the input VCF files contained at least one variant' logging.error(error_message) raise Error(error_message) if self.total_splits is not None or self.variants_per_split is not None or self.alleles_per_split is not None or os.path.exists( os.path.join(self.split_input_dir, 'data.pickle')): self._run_gramtools_with_split_vcf() else: self._run_gramtools_not_split_vcf() logging.info('Making plots from final.vcf') plots.plots_from_minos_vcf(self.final_vcf, self.plots_prefix) logging.info('All done! Thank you for using minos :)')
def run(self): self._make_output_dir() fh = logging.FileHandler(self.log_file, mode="w") log = logging.getLogger() formatter = logging.Formatter( "[minos %(asctime)s %(levelname)s] %(message)s", datefmt="%d-%m-%Y %H:%M:%S") fh.setFormatter(formatter) log.addHandler(fh) dependencies.check_and_report_dependencies(programs=["nextflow"]) self._prepare_nextflow_input_files() original_dir = os.getcwd() os.chdir(self.output_dir) nextflow_script = "nextflow.run.nf" MultiSamplePipeline._write_nextflow_script(nextflow_script) logging.info("Prepared nextflow files. cd " + self.output_dir) nextflow = dependencies.find_binary("nextflow") nextflow_command = [ nextflow, "run", "-work-dir", self.nextflow_work_dir, "-with-dag", "nextflow.out.dag.pdf", "-with-trace", "nextflow.out.trace.txt", ] if self.nextflow_config_file is not None: nextflow_command.extend(["-c", self.nextflow_config_file]) nextflow_command += [ nextflow_script, "--ref_fasta", self.ref_fasta, "--data_in_tsv", self.nextflow_input_tsv, "--max_alleles_per_cluster", str(self.max_alleles_per_cluster), "--min_large_ref_length", str(self.min_large_ref_length), "--final_outdir", self.output_dir, "--gramtools_max_read_length", str(self.gramtools_max_read_length), "--cluster_small_vars_ram", str(self.nf_ram_cluster_small_vars), "--gramtools_build_small_vars_ram", str(self.nf_ram_gramtools_build_small), "--gramtools_kmer_size", str(self.gramtools_kmer_size), "--gramtools_build_threads", str(self.gramtools_build_threads), "--minos_small_vars_ram", str(self.nf_ram_minos_small_vars), "--merge_small_vars_ram", str(self.nf_ram_merge_small_vars), ] if self.testing: nextflow_command.append("--testing") if self.use_unmapped_reads: nextflow_command.append("--use_unmapped_reads") if self.variants_per_split is not None: nextflow_command.append("--variants_per_split " + str(self.variants_per_split)) if self.alleles_per_split is not None: nextflow_command.append("--alleles_per_split " + str(self.alleles_per_split)) elif self.total_splits is not None: nextflow_command.append("--total_splits " + str(self.total_splits)) nextflow_command = " ".join(nextflow_command) if self.no_run: print( "Prepared nextflow pipeline. --no_run used, so not running. The nextflow command to run is:" ) print(nextflow_command) return else: logging.info("Start running nextflow: " + nextflow_command) syscall_process = utils.syscall(nextflow_command) logging.info( "Finish running nextflow. Writing nextflow stdout/stderr to files" ) with open("nextflow.stdout", "w") as f: print(syscall_process.stdout.rstrip(), file=f) with open("nextflow.stderr", "w") as f: print(syscall_process.stderr.rstrip(), file=f) logging.info("cd " + original_dir) if self.clean: logging.info("Delete nextflow work directory " + self.nextflow_work_dir) shutil.rmtree(self.nextflow_work_dir) logging.info("Delete .nextflow directory") shutil.rmtree(".nextflow") logging.info("Rename .nextflow.log -> nextflow.log") os.rename(".nextflow.log", "nextflow.log") os.chdir(original_dir)
def run(self): self.build_output_dir() fh = logging.FileHandler(self.log_file, mode="w") log = logging.getLogger() formatter = logging.Formatter( "[minos %(asctime)s %(levelname)s] %(message)s", datefmt="%d-%m-%Y %H:%M:%S") fh.setFormatter(formatter) log.addHandler(fh) logging.info("Command run: " + " ".join(sys.argv)) to_check = [ "gramtools", "vcfbreakmulti", "vcfallelicprimitives", "vcfuniq", "vt", ] dependencies.check_and_report_dependencies(programs=to_check) logging.info("Dependencies look OK") self.ref_fasta = os.path.join(self.outdir, "ref.fa") utils.fasta_to_upper_and_ACGT_only(self.original_ref_fasta, self.ref_fasta) if self.read_error_rate is None: logging.info( "read_error_rate unknown. Estimate from first 10,000 reads...") ( estimated_read_length, estimated_read_error_rate, ) = utils.estimate_max_read_length_and_read_error_rate_from_qual_scores( self.reads_files[0]) logging.info( f"Estimated read_error_rate={estimated_read_error_rate}") self.read_error_rate = (estimated_read_error_rate if self.read_error_rate is None else self.read_error_rate) logging.info(f"Using read_error_rate={self.read_error_rate}") if self.user_supplied_gramtools_build_dir: logging.info( "User supplied gramtools build dir. Assuming VCF already clustered, so skipping clustering" ) assert len(self.vcf_files) == 1 self.clustered_vcf = self.vcf_files[0] elif not self.cluster_input_vcfs: logging.info( "Skipping VCF clustering because user requested to skip") else: logging.info( "Clustering VCF file(s), to make one VCF input file for gramtools" ) tracker = variant_tracking.VariantTracker(self.cluster_dir, self.ref_fasta) tracker.merge_vcf_files(self.vcf_files) tracker.cluster(self.clustered_vcf_prefix, float("Inf"), max_alleles=5000) if not self.debug: os.unlink(f"{self.clustered_vcf_prefix}.excluded.tsv") utils.rm_rf(self.cluster_dir) logging.info("Finished clustering VCF file(s)") if not vcf_file_read.vcf_file_has_at_least_one_record( self.clustered_vcf): error_message = "No VCF records. Cannot continue. Please check that the input VCF files contained at least one variant" logging.error(error_message) raise Exception(error_message) if (self.total_splits is not None or self.variants_per_split is not None or self.alleles_per_split is not None or os.path.exists( os.path.join(self.split_input_dir, "data.pickle"))): self._run_gramtools_with_split_vcf() else: self._run_gramtools_not_split_vcf() logging.info("All done! Thank you for using minos :)")
def run(self): self.build_output_dir() fh = logging.FileHandler(self.log_file, mode="w") log = logging.getLogger() formatter = logging.Formatter( "[minos %(asctime)s %(levelname)s] %(message)s", datefmt="%d-%m-%Y %H:%M:%S") fh.setFormatter(formatter) log.addHandler(fh) logging.info("Command run: " + " ".join(sys.argv)) dependencies.check_and_report_dependencies(programs=["gramtools"]) logging.info("Dependencies look OK") if self.read_error_rate is None or self.max_read_length is None: logging.info( "One or both of read_error_rate and max_read_length not known. Estimate from first 10,000 reads..." ) estimated_read_length, estimated_read_error_rate = utils.estimate_max_read_length_and_read_error_rate_from_qual_scores( self.reads_files[0]) logging.info("Estimated max_read_length=" + str(estimated_read_length) + " and read_error_rate=" + str(estimated_read_error_rate)) self.read_error_rate = (estimated_read_error_rate if self.read_error_rate is None else self.read_error_rate) self.max_read_length = (estimated_read_length if self.max_read_length is None else self.max_read_length) logging.info("Using max_read_length=" + str(self.max_read_length) + " and read_error_rate=" + str(self.read_error_rate)) if self.user_supplied_gramtools_build_dir: logging.info( "User supplied gramtools build dir. Assuming VCF already clustered, so skipping clustering" ) assert len(self.vcf_files) == 1 self.clustered_vcf = self.vcf_files[0] else: logging.info( "Clustering VCF file(s), to make one VCF input file for gramtools" ) clusterer = vcf_clusterer.VcfClusterer( self.vcf_files, self.ref_fasta, self.clustered_vcf, cluster_boundary_size=0, max_alleles_per_cluster=self.max_alleles_per_cluster, ) clusterer.run() logging.info("Finished clustering VCF file(s)") if not vcf_file_read.vcf_file_has_at_least_one_record( self.clustered_vcf): error_message = "No VCF records. Cannot continue. Please check that the input VCF files contained at least one variant" logging.error(error_message) raise Exception(error_message) if (self.total_splits is not None or self.variants_per_split is not None or self.alleles_per_split is not None or os.path.exists( os.path.join(self.split_input_dir, "data.pickle"))): self._run_gramtools_with_split_vcf() else: self._run_gramtools_not_split_vcf() logging.info("Making plots from final.vcf") plots.plots_from_minos_vcf(self.final_vcf, self.plots_prefix) logging.info("All done! Thank you for using minos :)")