def cgmlst_profiles_df(fastas, cgmlst_results): genome_marker_cgmlst_result = {} for fasta, res in zip(fastas, cgmlst_results): genome = genome_name_from_fasta_path(fasta) tmp = {} for marker, res_dict in res.items(): aname = res_dict['name'] tmp[marker] = int(aname) if aname is not None else None genome_marker_cgmlst_result[genome] = tmp return pd.DataFrame(genome_marker_cgmlst_result).transpose()
def cgmlst_profiles_df(fastas, cgmlst_results): genome_marker_cgmlst_result = {} for fasta, res in zip(fastas, cgmlst_results): genome = genome_name_from_fasta_path(fasta) tmp = {} for marker, res_dict in res.items(): aname = res_dict['name'] tmp[marker] = int(aname) if aname is not None else None genome_marker_cgmlst_result[genome] = tmp return pd.DataFrame(genome_marker_cgmlst_result).transpose()
def sketch_fasta(fasta_path, outdir): """Create a Mash sketch from an input fasta file Args: fasta_path (str): input fasta file path. Genome name in fasta filename outdir (str): output directory path to write Mash sketch file to Returns: str: output Mash sketch file path """ genome_name = genome_name_from_fasta_path(fasta_path) outpath = os.path.join(outdir, genome_name) args = ['mash', 'sketch', '-o', outpath, fasta_path] logging.info('Running Mash sketch with command: %s', ' '.join(args)) p = Popen(args) p.wait() sketch_path = outpath + '.msh' assert os.path.exists(sketch_path), 'Mash sketch for genome {} was not created at {}'.format( genome_name, sketch_path) return sketch_path
def sketch_fasta(fasta_path, outdir): """Create a Mash sketch from an input fasta file Args: fasta_path (str): input fasta file path. Genome name in fasta filename outdir (str): output directory path to write Mash sketch file to Returns: str: output Mash sketch file path """ genome_name = genome_name_from_fasta_path(fasta_path) outpath = os.path.join(outdir, genome_name) args = ['mash', 'sketch', '-o', outpath, fasta_path] logging.info('Running Mash sketch with command: %s', ' '.join(args)) p = Popen(args) p.wait() sketch_path = outpath + '.msh' assert os.path.exists( sketch_path), 'Mash sketch for genome {} was not created at {}'.format( genome_name, sketch_path) return sketch_path
def run_sistr(input_fasta, tmp_dir): blast_runner = None try: assert os.path.exists( input_fasta), "Input fasta file '%s' must exist!" % input_fasta fasta_filename = os.path.basename(input_fasta) genome_name = genome_name_from_fasta_path(input_fasta) dtnow = datetime.now() genome_tmp_dir = os.path.join( tmp_dir, dtnow.strftime("%Y%m%d%H%M%S") + '-' + 'SISTR' + '-' + genome_name) blast_runner = BlastRunner(input_fasta, genome_tmp_dir) logging.info( 'Initializing temporary analysis directory "%s" and preparing for BLAST searching.', genome_tmp_dir) blast_runner.prep_blast() logging.info('Temporary FASTA file copied to %s', blast_runner.tmp_fasta_path) cgmlst_prediction, cgmlst_results = run_cgmlst(blast_runner) spp = cgmlst_prediction['subspecies'] serovar_predictor = SerovarPredictor(blast_runner, spp) serovar_predictor.predict_serovar_from_antigen_blast() prediction = serovar_predictor.get_serovar_prediction() merge_cgmlst_prediction(prediction, cgmlst_prediction) overall_serovar_call(prediction, serovar_predictor) logging.info( '%s | Antigen gene BLAST serovar prediction: "%s" serogroup=%s:H1=%s:H2=%s', fasta_filename, prediction.serovar_antigen, prediction.serogroup, prediction.h1, prediction.h2) logging.info('%s | Subspecies prediction: %s', fasta_filename, spp) logging.info('%s | Overall serovar prediction: %s', fasta_filename, prediction.serovar) finally: logging.info('Deleting temporary working directory at %s', blast_runner.tmp_work_dir) blast_runner.cleanup() return prediction, cgmlst_results
def run_sistr(input_fasta, tmp_dir): blast_runner = None try: assert os.path.exists(input_fasta), "Input fasta file '%s' must exist!" % input_fasta fasta_filename = os.path.basename(input_fasta) genome_name = genome_name_from_fasta_path(input_fasta) dtnow = datetime.now() genome_tmp_dir = os.path.join(tmp_dir, dtnow.strftime("%Y%m%d%H%M%S") + '-' + 'SISTR' + '-' + genome_name) blast_runner = BlastRunner(input_fasta, genome_tmp_dir) logging.info('Initializing temporary analysis directory "%s" and preparing for BLAST searching.', genome_tmp_dir) blast_runner.prep_blast() logging.info('Temporary FASTA file copied to %s', blast_runner.tmp_fasta_path) cgmlst_prediction, cgmlst_results = run_cgmlst(blast_runner) spp = cgmlst_prediction['subspecies'] serovar_predictor = SerovarPredictor(blast_runner, spp) serovar_predictor.predict_serovar_from_antigen_blast() prediction = serovar_predictor.get_serovar_prediction() merge_cgmlst_prediction(prediction, cgmlst_prediction) overall_serovar_call(prediction, serovar_predictor) logging.info('%s | Antigen gene BLAST serovar prediction: "%s" serogroup=%s:H1=%s:H2=%s', fasta_filename, prediction.serovar_antigen, prediction.serogroup, prediction.h1, prediction.h2) logging.info('%s | Subspecies prediction: %s', fasta_filename, spp) logging.info('%s | Overall serovar prediction: %s', fasta_filename, prediction.serovar) finally: logging.info('Deleting temporary working directory at %s', blast_runner.tmp_work_dir) blast_runner.cleanup() return prediction, cgmlst_results
def main(): parser = init_parser() args = parser.parse_args() init_console_logger(args.verbose) logging.debug(args) input_fastas = args.fastas outdir = args.outdir tmp_dir = args.tmp_dir serovar_table_path = args.serovar_table threads = args.threads force = args.force assert len(input_fastas) > 0, 'No FASTA files specified!' for input_fasta in input_fastas: assert os.path.exists(input_fasta), 'Genome FASTA file does not exist at "{}"'.format(input_fasta) genome_names = [genome_name_from_fasta_path(x) for x in input_fastas] logging.info('You have specified %s genomes to add to current sistr_cmd data files! %s', len(genome_names), genome_names) if os.path.exists(outdir): if not force: raise Exception('Output directory already exists at {}!'.format(outdir)) else: logging.warning('Using existing output directory at %s', outdir) try: os.makedirs(outdir) except: pass assert os.path.exists(outdir), 'Output directory could not be created!' if serovar_table_path: assert os.path.exists(serovar_table_path), 'Provided serovar table path does not exist! {}'.format( serovar_table_path) logging.info('Parsing serovar table from "%s"', serovar_table_path) if re.match(r'.*.csv$', serovar_table_path): logging.info('Trying to read serovar table "%s" as CSV', serovar_table_path) df_serovar = pd.read_csv(serovar_table_path) else: logging.info('Trying to read serovar table "%s" as tab-delimited', serovar_table_path) df_serovar = pd.read_table(serovar_table_path) expected_columns = ['genome', 'serovar'] assert np.all( df_serovar.columns.isin(expected_columns)), 'User serovar table did not contain expected columns {}'.format( expected_columns) if 'subspecies' not in df_serovar.columns: logging.warning( 'User serovar table did not contain "subspecies" column so the sistr_cmd subspecies prediction will be used!') genome_names_series = pd.Series(genome_names) genomes_in_serovar_table = genome_names_series.isin(df_serovar.genome) if not np.all(genomes_in_serovar_table): missing_genomes = ', '.join([x for x in genome_names_series[~genomes_in_serovar_table]]) logging.error('The following genomes were not found in the serovar table: %s', missing_genomes) raise Exception('Not all user provided genome FASTA files in the provided serovar table!') df_wklm = pd.read_csv(SEROVAR_TABLE_PATH) logging.info('Checking for non-standard serovar designations') serovars_not_in_wklm = df_serovar.serovar[~df_serovar.serovar.isin(df_wklm.Serovar)] for row_idx, serovar in serovars_not_in_wklm.iteritems(): logging.warning('Non-standard serovar %s at row %s for genome %s!', serovar, row_idx, df_serovar.ix[row_idx]['genome']) else: logging.warning('No genome to serovar table specified! Using SISTR serovar predictions') df_serovar = None if threads == 1: logging.info('Serial single threaded run mode on %s genomes', len(input_fastas)) outputs = [run_sistr(input_fasta, tmp_dir) for input_fasta in input_fastas] else: from multiprocessing import Pool logging.info('Initializing thread pool with %s threads', threads) pool = Pool(processes=threads) logging.info('Running SISTR analysis asynchronously on %s genomes', len(input_fastas)) res = [pool.apply_async(run_sistr, (input_fasta, tmp_dir)) for input_fasta in input_fastas] logging.info('Getting SISTR analysis results') outputs = [x.get() for x in res] # collect results from sistr analysis prediction_outputs = [x for x, y in outputs] cgmlst_results = [y for x, y in outputs] # create some output dirs data_outdir = create_subdirs(outdir, 'data') cgmlst_outdir = create_subdirs(outdir, 'data', 'cgmlst') sketch_outdir = create_subdirs(outdir, 'mash-sketches') # write files with new and old data cgmlst_fasta = write_cgmlst_fasta(cgmlst_outdir, cgmlst_results) write_cgmlst_profiles_csv(cgmlst_outdir, cgmlst_results, genome_names) write_serovar_and_spp_tables(data_outdir, df_serovar, prediction_outputs, genome_names) create_merge_mash_sketches(input_fastas, data_outdir, sketch_outdir) centroid_alleles_path = os.path.join(cgmlst_outdir, 'cgmlst-centroid.fasta') run_allele_reduction(cgmlst_fasta, centroid_alleles_path, threads=threads) logging.info('Done!')
def main(): parser = init_parser() args = parser.parse_args() init_console_logger(args.verbose) logging.debug(args) input_fastas = args.fastas outdir = args.outdir tmp_dir = args.tmp_dir serovar_table_path = args.serovar_table threads = args.threads force = args.force assert len(input_fastas) > 0, 'No FASTA files specified!' for input_fasta in input_fastas: assert os.path.exists( input_fasta), 'Genome FASTA file does not exist at "{}"'.format( input_fasta) genome_names = [genome_name_from_fasta_path(x) for x in input_fastas] logging.info( 'You have specified %s genomes to add to current sistr_cmd data files! %s', len(genome_names), genome_names) if os.path.exists(outdir): if not force: raise Exception( 'Output directory already exists at {}!'.format(outdir)) else: logging.warning('Using existing output directory at %s', outdir) try: os.makedirs(outdir) except: pass assert os.path.exists(outdir), 'Output directory could not be created!' if serovar_table_path: assert os.path.exists( serovar_table_path ), 'Provided serovar table path does not exist! {}'.format( serovar_table_path) logging.info('Parsing serovar table from "%s"', serovar_table_path) if re.match(r'.*.csv$', serovar_table_path): logging.info('Trying to read serovar table "%s" as CSV', serovar_table_path) df_serovar = pd.read_csv(serovar_table_path) else: logging.info('Trying to read serovar table "%s" as tab-delimited', serovar_table_path) df_serovar = pd.read_table(serovar_table_path) expected_columns = ['genome', 'serovar'] assert np.all( df_serovar.columns.isin(expected_columns) ), 'User serovar table did not contain expected columns {}'.format( expected_columns) if 'subspecies' not in df_serovar.columns: logging.warning( 'User serovar table did not contain "subspecies" column so the sistr_cmd subspecies prediction will be used!' ) genome_names_series = pd.Series(genome_names) genomes_in_serovar_table = genome_names_series.isin(df_serovar.genome) if not np.all(genomes_in_serovar_table): missing_genomes = ', '.join( [x for x in genome_names_series[~genomes_in_serovar_table]]) logging.error( 'The following genomes were not found in the serovar table: %s', missing_genomes) raise Exception( 'Not all user provided genome FASTA files in the provided serovar table!' ) df_wklm = pd.read_csv(SEROVAR_TABLE_PATH) logging.info('Checking for non-standard serovar designations') serovars_not_in_wklm = df_serovar.serovar[~df_serovar.serovar. isin(df_wklm.Serovar)] for row_idx, serovar in serovars_not_in_wklm.iteritems(): logging.warning('Non-standard serovar %s at row %s for genome %s!', serovar, row_idx, df_serovar.ix[row_idx]['genome']) else: logging.warning( 'No genome to serovar table specified! Using SISTR serovar predictions' ) df_serovar = None if threads == 1: logging.info('Serial single threaded run mode on %s genomes', len(input_fastas)) outputs = [ run_sistr(input_fasta, tmp_dir) for input_fasta in input_fastas ] else: from multiprocessing import Pool logging.info('Initializing thread pool with %s threads', threads) pool = Pool(processes=threads) logging.info('Running SISTR analysis asynchronously on %s genomes', len(input_fastas)) res = [ pool.apply_async(run_sistr, (input_fasta, tmp_dir)) for input_fasta in input_fastas ] logging.info('Getting SISTR analysis results') outputs = [x.get() for x in res] # collect results from sistr analysis prediction_outputs = [x for x, y in outputs] cgmlst_results = [y for x, y in outputs] # create some output dirs data_outdir = create_subdirs(outdir, 'data') cgmlst_outdir = create_subdirs(outdir, 'data', 'cgmlst') sketch_outdir = create_subdirs(outdir, 'mash-sketches') # write files with new and old data cgmlst_fasta = write_cgmlst_fasta(cgmlst_outdir, cgmlst_results) write_cgmlst_profiles_csv(cgmlst_outdir, cgmlst_results, genome_names) write_serovar_and_spp_tables(data_outdir, df_serovar, prediction_outputs, genome_names) create_merge_mash_sketches(input_fastas, data_outdir, sketch_outdir) centroid_alleles_path = os.path.join(cgmlst_outdir, 'cgmlst-centroid.fasta') run_allele_reduction(cgmlst_fasta, centroid_alleles_path, threads=threads) logging.info('Done!')