def CF_bam2RPKM(args): try: import pysam except: print '[ERROR] Cannot load pysam module! Make sure it is insalled' sys.exit(0) try: # read probes table probe_fn = str(args.probes[0]) probes = cf.loadProbeList(probe_fn) num_probes = len(probes) print '[INIT] Successfully read in %d probes from %s' % (num_probes, probe_fn) except IOError as e: print '[ERROR] Cannot read probes file: ', probe_fn sys.exit(0) try: rpkm_f = open(args.output[0], 'w') except IOError as e: print '[ERROR] Cannot open rpkm file for writing: ', args.output sys.exit(0) print "[RUNNING] Counting total number of reads in bam file..." total_reads = float(pysam.view("-c", args.input[0])[0].strip("\n")) print "[RUNNING] Found %d reads" % total_reads f = pysam.Samfile(args.input[0], "rb") if not f.has_Index(): print "[ERROR] No index found for bam file (%s)!\n[ERROR] You must first index the bam file and include the .bai file in the same directory as the bam file!" % args.input[ 0] sys.exit(0) # will be storing values in these arrays readcount = np.zeros(num_probes) exon_bp = np.zeros(num_probes) probeIDs = np.zeros(num_probes) counter = 0 # detect contig naming scheme here # TODO, add an optional "contigs.txt" file or automatically handle contig naming bam_contigs = f.references probes_contigs = [ str(p) for p in set(map(operator.itemgetter("chr"), probes)) ] probes2contigmap = {} for probes_contig in probes_contigs: if probes_contig in bam_contigs: probes2contigmap[probes_contig] = probes_contig elif cf.chrInt2Str(probes_contig) in bam_contigs: probes2contigmap[probes_contig] = cf.chrInt2Str(probes_contig) elif cf.chrInt2Str(probes_contig).replace("chr", "") in bam_contigs: probes2contigmap[probes_contig] = cf.chrInt2Str( probes_contig).replace("chr", "") else: print "[ERROR] Could not find contig '%s' from %s in bam file! \n[ERROR] Perhaps the contig names for the probes are incompatible with the bam file ('chr1' vs. '1'), or unsupported contig naming is used?" % ( probes_contig, probe_fn) sys.exit(0) print "[RUNNING] Calculating RPKM values..." # loop through each probe for p in probes: # f.fetch is a pysam method and returns an iterator for reads overlapping interval p_chr = probes2contigmap[str(p["chr"])] p_start = p["start"] p_stop = p["stop"] try: iter = f.fetch(p_chr, p_start, p_stop) except: print "[ERROR] Could not retrieve mappings for region %s:%d-%d. Check that contigs are named correctly and the bam file is properly indexed" % ( p_chr, p_start, p_stop) sys.exit(0) for i in iter: if i.pos + 1 >= p_start: #this checks to make sure a read actually starts in an interval readcount[counter] += 1 exon_bp[counter] = p_stop - p_start probeIDs[counter] = counter + 1 #probeIDs are 1-based counter += 1 #calcualte RPKM values for all probes rpkm = (10**9 * (readcount) / (exon_bp)) / (total_reads) out = np.vstack([probeIDs, readcount, rpkm]) np.savetxt(rpkm_f, out.transpose(), delimiter='\t', fmt=['%d', '%d', '%f']) rpkm_f.close()
def CF_bam2RPKM(args): try: import pysam except: print '[ERROR] Cannot load pysam module! Make sure it is insalled' sys.exit(0) try: # read probes table probe_fn = str(args.probes[0]) probes = cf.loadProbeList(probe_fn) num_probes = len(probes) print '[INIT] Successfully read in %d probes from %s' % (num_probes, probe_fn) except IOError as e: print '[ERROR] Cannot read probes file: ', probe_fn sys.exit(0) try: rpkm_f = open(args.output[0],'w') except IOError as e: print '[ERROR] Cannot open rpkm file for writing: ', args.output sys.exit(0) print "[RUNNING] Counting total number of reads in bam file..." total_reads = float(pysam.view("-c", args.input[0])[0].strip("\n")) print "[RUNNING] Found %d reads" % total_reads f = pysam.Samfile(args.input[0], "rb" ) if not f._hasIndex(): print "[ERROR] No index found for bam file (%s)!\n[ERROR] You must first index the bam file and include the .bai file in the same directory as the bam file!" % args.input[0] sys.exit(0) # will be storing values in these arrays readcount = np.zeros(num_probes) exon_bp = np.zeros(num_probes) probeIDs = np.zeros(num_probes) counter = 0 # detect contig naming scheme here # TODO, add an optional "contigs.txt" file or automatically handle contig naming bam_contigs = f.references probes_contigs = [str(p) for p in set(map(operator.itemgetter("chr"),probes))] probes2contigmap = {} for probes_contig in probes_contigs: if probes_contig in bam_contigs: probes2contigmap[probes_contig] = probes_contig elif cf.chrInt2Str(probes_contig) in bam_contigs: probes2contigmap[probes_contig] = cf.chrInt2Str(probes_contig) elif cf.chrInt2Str(probes_contig).replace("chr","") in bam_contigs: probes2contigmap[probes_contig] = cf.chrInt2Str(probes_contig).replace("chr","") else: print "[ERROR] Could not find contig '%s' from %s in bam file! \n[ERROR] Perhaps the contig names for the probes are incompatible with the bam file ('chr1' vs. '1'), or unsupported contig naming is used?" % (probes_contig, probe_fn) sys.exit(0) print "[RUNNING] Calculating RPKM values..." # loop through each probe for p in probes: # f.fetch is a pysam method and returns an iterator for reads overlapping interval p_chr = probes2contigmap[str(p["chr"])] p_start = p["start"] p_stop = p["stop"] try: iter = f.fetch(p_chr,p_start,p_stop) except: print "[ERROR] Could not retrieve mappings for region %s:%d-%d. Check that contigs are named correctly and the bam file is properly indexed" % (p_chr,p_start,p_stop) sys.exit(0) for i in iter: if i.pos+1 >= p_start: #this checks to make sure a read actually starts in an interval readcount[counter] += 1 exon_bp[counter] = p_stop-p_start probeIDs[counter] = counter +1 #probeIDs are 1-based counter +=1 #calcualte RPKM values for all probes rpkm = (10**9*(readcount)/(exon_bp))/(total_reads) out = np.vstack([probeIDs,readcount,rpkm]) np.savetxt(rpkm_f,out.transpose(),delimiter='\t',fmt=['%d','%d','%f']) rpkm_f.close()
def CF_analyze(args): # do path/file checks: try: # read probes table probe_fn = str(args.probes[0]) probes = cf.loadProbeList(probe_fn) num_probes = len(probes) print '[INIT] Successfully read in %d probes from %s' % (num_probes, probe_fn) except IOError as e: print '[ERROR] Cannot read probes file: ', probe_fn sys.exit(0) try: svd_outfile_fn = str(args.output) h5file_out = openFile(svd_outfile_fn, mode='w') probe_group = h5file_out.createGroup("/", "probes", "probes") except IOError as e: print '[ERROR] Cannot open SVD output file for writing: ', svd_outfile_fn sys.exit(0) if args.write_svals != "": sval_f = open(args.write_svals, 'w') if args.plot_scree != "": try: import matplotlib matplotlib.use('Agg') import matplotlib.pyplot as plt import pylab as P from matplotlib.lines import Line2D from matplotlib.patches import Rectangle except: print "[ERROR] One or more of the required modules for plotting cannot be loaded! Are matplotlib and pylab installed?" sys.exit(0) plt.gcf().clear() fig = plt.figure(figsize=(10, 5)) ax = fig.add_subplot(111) rpkm_dir = str(args.rpkm_dir[0]) rpkm_files = glob.glob(rpkm_dir + "/*") if len(rpkm_files) == 0: print '[ERROR] Cannot find any files in RPKM directory (or directory path is incorrect): ', rpkm_dir sys.exit(0) elif len(rpkm_files) == 1: print '[ERROR] Found only 1 RPKM file (sample). CoNIFER requires multiple samples (8 or more) to run. Exiting.' sys.exit(0) elif len(rpkm_files) < 8: print '[WARNING] Only found %d samples... this is less than the recommended minimum, and CoNIFER may not analyze this dataset correctly!' % len( rpkm_files) elif len(rpkm_files) <= int(args.svd): print '[ERROR] The number of SVD values specified (%d) must be less than the number of samples (%d). Either add more samples to the analysis or reduce the --svd parameter! Exiting.' % ( len(rpkm_files), int(args.svd)) sys.exit(0) else: print '[INIT] Found %d RPKM files in %s' % (len(rpkm_files), rpkm_dir) # read in samples names and generate file list samples = {} for f in rpkm_files: s = '.'.join(f.split('/')[-1].split('.')[0:-1]) print "[INIT] Mapping file to sampleID: %s --> %s" % (f, s) samples[s] = f #check uniqueness and total # of samples if len(set(samples)) != len(set(rpkm_files)): print '[ERROR] Could not successfully derive sample names from RPKM filenames. There are probably non-unique sample names! Please rename files using <sampleID>.txt format!' sys.exit(0) # LOAD RPKM DATA RPKM_data = np.zeros([num_probes, len(samples)], dtype=np.float) failed_samples = 0 for i, s in enumerate(samples.keys()): t = np.loadtxt(samples[s], dtype=np.float, delimiter="\t", skiprows=0, usecols=[2]) if len(t) != num_probes: print "[WARNING] Number of RPKM values for %s in file %s does not match number of defined probes in %s. **This sample will be dropped from analysis**!" % ( s, samples[s], probe_fn) _ = samples.pop(s) failed_samples += 1 else: RPKM_data[:, i] = t print "[INIT] Successfully read RPKM data for sampleID: %s" % s RPKM_data = RPKM_data[:, 0:len(samples)] print "[INIT] Finished reading RPKM files. Total number of samples in experiment: %d (%d failed to read properly)" % ( len(samples), failed_samples) if len(samples) <= int(args.svd): print '[ERROR] The number of SVD values specified (%d) must be less than the number of samples (%d). Either add more samples to the analysis or reduce the --svd parameter! Exiting.' % ( int(args.svd), len(samples)) sys.exit(0) # BEGIN chrs_to_process = set(map(operator.itemgetter("chr"), probes)) chrs_to_process_str = ', '.join( [cf.chrInt2Str(c) for c in chrs_to_process]) print '[INIT] Attempting to process chromosomes: ', chrs_to_process_str for chr in chrs_to_process: print "[RUNNING: chr%d] Now on: %s" % (chr, cf.chrInt2Str(chr)) chr_group_name = "chr%d" % chr chr_group = h5file_out.createGroup("/", chr_group_name, chr_group_name) chr_probes = filter(lambda i: i["chr"] == chr, probes) num_chr_probes = len(chr_probes) start_probeID = chr_probes[0]['probeID'] stop_probeID = chr_probes[-1]['probeID'] print "[RUNNING: chr%d] Found %d probes; probeID range is [%d-%d]" % ( chr, len(chr_probes), start_probeID - 1, stop_probeID ) # probeID is 1-based and slicing is 0-based, hence the start_probeID-1 term rpkm = RPKM_data[start_probeID:stop_probeID, :] print "[RUNNING: chr%d] Calculating median RPKM" % chr median = np.median(rpkm, 1) sd = np.std(rpkm, 1) probe_mask = median >= float(args.min_rpkm) print "[RUNNING: chr%d] Masking %d probes with median RPKM < %f" % ( chr, np.sum(probe_mask == False), float(args.min_rpkm)) rpkm = rpkm[probe_mask, :] num_chr_probes = np.sum(probe_mask) if num_chr_probes <= len(samples): print "[ERROR] This chromosome has fewer informative probes than there are samples in the analysis! There are probably no mappings on this chromosome. Please remove these probes from the probes.txt file" sys.exit(0) probeIDs = np.array(map(operator.itemgetter("probeID"), chr_probes))[probe_mask] probe_starts = np.array(map(operator.itemgetter("start"), chr_probes))[probe_mask] probe_stops = np.array(map(operator.itemgetter("stop"), chr_probes))[probe_mask] gene_names = np.array(map(operator.itemgetter("name"), chr_probes))[probe_mask] dt = np.dtype([('probeID', np.uint32), ('start', np.uint32), ('stop', np.uint32), ('name', np.str_, 20)]) out_probes = np.empty(num_chr_probes, dtype=dt) out_probes['probeID'] = probeIDs out_probes['start'] = probe_starts out_probes['stop'] = probe_stops out_probes['name'] = gene_names probe_table = h5file_out.createTable(probe_group, "probes_chr%d" % chr, cf.probe, "chr%d" % chr) probe_table.append(out_probes) print "[RUNNING: chr%d] Calculating ZRPKM scores..." % chr rpkm = np.apply_along_axis(cf.zrpkm, 0, rpkm, median[probe_mask], sd[probe_mask]) # svd transform print "[RUNNING: chr%d] SVD decomposition..." % chr components_removed = int(args.svd) U, S, Vt = np.linalg.svd(rpkm, full_matrices=False) new_S = np.diag( np.hstack([np.zeros([components_removed]), S[components_removed:]])) if args.write_svals != "": sval_f.write('chr' + str(chr) + '\t' + '\t'.join([str(_i) for _i in S]) + "\n") if args.plot_scree != "": ax.plot(S, label='chr' + str(chr), lw=0.5) # reconstruct data matrix rpkm = np.dot(U, np.dot(new_S, Vt)) # save to HDF5 file print "[RUNNING: chr%d] Saving SVD-ZRPKM values" % chr for i, s in enumerate(samples): out_data = np.empty(num_chr_probes, dtype='u4,f8') out_data['f0'] = probeIDs out_data['f1'] = rpkm[:, i] sample_tbl = h5file_out.createTable(chr_group, "sample_" + str(s), cf.rpkm_value, "%s" % str(s)) sample_tbl.append(out_data) print "[RUNNING] Saving sampleIDs to file..." sample_group = h5file_out.createGroup("/", "samples", "samples") sample_table = h5file_out.createTable(sample_group, "samples", cf.sample, "samples") dt = np.dtype([('sampleID', np.str_, 100)]) out_samples = np.empty(len(samples.keys()), dtype=dt) out_samples['sampleID'] = np.array(samples.keys()) sample_table.append(out_samples) if args.write_sd != "": print "[RUNNING] Calculating standard deviations for all samples (this can take a while)..." sd_file = open(args.write_sd, 'w') for i, s in enumerate(samples): # collect all SVD-ZRPKM values count = 1 for chr in chrs_to_process: if count == 1: sd_out = h5file_out.root._f_getChild( "chr%d" % chr)._f_getChild( "sample_%s" % s).read(field="rpkm").flatten() else: sd_out = np.hstack([ sd_out, out.h5file_out.root._f_getChild( "chr%d" % chr)._f_getChild( "sample_%s" % s).read(field="rpkm").flatten() ]) sd = np.std(sd_out) sd_file.write("%s\t%f\n" % (s, sd)) sd_file.close() if args.plot_scree != "": plt.title("Scree plot") if len(samples) < 50: plt.xlim([0, len(samples)]) plt.xlabel("S values") else: plt.xlim([0, 50]) plt.xlabel("S values (only first 50 plotted)") plt.ylabel("Relative contributed variance") plt.savefig(args.plot_scree) print "[FINISHED]" h5file_out.close() sys.exit(0)
def CF_analyze(args): # do path/file checks: try: # read probes table probe_fn = str(args.probes[0]) probes = cf.loadProbeList(probe_fn) num_probes = len(probes) print '[INIT] Successfully read in %d probes from %s' % (num_probes, probe_fn) except IOError as e: print '[ERROR] Cannot read probes file: ', probe_fn sys.exit(0) try: svd_outfile_fn = str(args.output) h5file_out = openFile(svd_outfile_fn, mode='w') probe_group = h5file_out.createGroup("/","probes","probes") except IOError as e: print '[ERROR] Cannot open SVD output file for writing: ', svd_outfile_fn sys.exit(0) if args.write_svals != "": sval_f = open(args.write_svals,'w') if args.plot_scree != "": try: import matplotlib matplotlib.use('Agg') import matplotlib.pyplot as plt import pylab as P from matplotlib.lines import Line2D from matplotlib.patches import Rectangle except: print "[ERROR] One or more of the required modules for plotting cannot be loaded! Are matplotlib and pylab installed?" sys.exit(0) plt.gcf().clear() fig = plt.figure(figsize=(10,5)) ax = fig.add_subplot(111) rpkm_dir = str(args.rpkm_dir[0]) rpkm_files = glob.glob(rpkm_dir + "/*") if len(rpkm_files) == 0: print '[ERROR] Cannot find any files in RPKM directory (or directory path is incorrect): ', rpkm_dir sys.exit(0) elif len(rpkm_files) == 1: print '[ERROR] Found only 1 RPKM file (sample). CoNIFER requires multiple samples (8 or more) to run. Exiting.' sys.exit(0) elif len(rpkm_files) < 8: print '[WARNING] Only found %d samples... this is less than the recommended minimum, and CoNIFER may not analyze this dataset correctly!' % len(rpkm_files) elif len(rpkm_files) <= int(args.svd): print '[ERROR] The number of SVD values specified (%d) must be less than the number of samples (%d). Either add more samples to the analysis or reduce the --svd parameter! Exiting.' % (len(rpkm_files), int(args.svd)) sys.exit(0) else: print '[INIT] Found %d RPKM files in %s' % (len(rpkm_files), rpkm_dir) # read in samples names and generate file list samples = {} for f in rpkm_files: s = '.'.join(f.split('/')[-1].split('.')[0:-1]) print "[INIT] Mapping file to sampleID: %s --> %s" % (f, s) samples[s] = f #check uniqueness and total # of samples if len(set(samples)) != len(set(rpkm_files)): print '[ERROR] Could not successfully derive sample names from RPKM filenames. There are probably non-unique sample names! Please rename files using <sampleID>.txt format!' sys.exit(0) # LOAD RPKM DATA RPKM_data = np.zeros([num_probes,len(samples)], dtype=np.float) failed_samples = 0 for i,s in enumerate(samples.keys()): t = cf.loadRPKM(samples[s]) if len(t) != num_probes: print "[WARNING] Number of RPKM values for %s in file %s does not match number of defined probes in %s. **This sample will be dropped from analysis**!" % (s, samples[s], probe_fn) _ = samples.pop(s) failed_samples += 1 else: RPKM_data[:,i] = t print "[INIT] Successfully read RPKM data for sampleID: %s" % s RPKM_data = RPKM_data[:,0:len(samples)] print "[INIT] Finished reading RPKM files. Total number of samples in experiment: %d (%d failed to read properly)" % (len(samples), failed_samples) if len(samples) <= int(args.svd): print '[ERROR] The number of SVD values specified (%d) must be less than the number of samples (%d). Either add more samples to the analysis or reduce the --svd parameter! Exiting.' % (int(args.svd), len(samples)) sys.exit(0) # BEGIN chrs_to_process = set(map(operator.itemgetter("chr"),probes)) chrs_to_process_str = ', '.join([cf.chrInt2Str(c) for c in chrs_to_process]) print '[INIT] Attempting to process chromosomes: ', chrs_to_process_str for chr in chrs_to_process: print "[RUNNING: chr%d] Now on: %s" %(chr, cf.chrInt2Str(chr)) chr_group_name = "chr%d" % chr chr_group = h5file_out.createGroup("/",chr_group_name,chr_group_name) chr_probes = filter(lambda i: i["chr"] == chr, probes) num_chr_probes = len(chr_probes) start_probeID = chr_probes[0]['probeID'] stop_probeID = chr_probes[-1]['probeID'] print "[RUNNING: chr%d] Found %d probes; probeID range is [%d-%d]" % (chr, len(chr_probes), start_probeID-1, stop_probeID) # probeID is 1-based and slicing is 0-based, hence the start_probeID-1 term rpkm = RPKM_data[start_probeID:stop_probeID,:] print "[RUNNING: chr%d] Calculating median RPKM" % chr median = np.median(rpkm,1) sd = np.std(rpkm,1) probe_mask = median >= float(args.min_rpkm) print "[RUNNING: chr%d] Masking %d probes with median RPKM < %f" % (chr, np.sum(probe_mask==False), float(args.min_rpkm)) rpkm = rpkm[probe_mask, :] num_chr_probes = np.sum(probe_mask) if num_chr_probes <= len(samples): print "[ERROR] This chromosome has fewer informative probes than there are samples in the analysis! There are probably no mappings on this chromosome. Please remove these probes from the probes.txt file" sys.exit(0) probeIDs = np.array(map(operator.itemgetter("probeID"),chr_probes))[probe_mask] probe_starts = np.array(map(operator.itemgetter("start"),chr_probes))[probe_mask] probe_stops = np.array(map(operator.itemgetter("stop"),chr_probes))[probe_mask] gene_names = np.array(map(operator.itemgetter("name"),chr_probes))[probe_mask] dt = np.dtype([('probeID',np.uint32),('start',np.uint32),('stop',np.uint32), ('name', np.str_, 20)]) out_probes = np.empty(num_chr_probes,dtype=dt) out_probes['probeID'] = probeIDs out_probes['start'] = probe_starts out_probes['stop'] = probe_stops out_probes['name'] = gene_names probe_table = h5file_out.createTable(probe_group,"probes_chr%d" % chr,cf.probe,"chr%d" % chr) probe_table.append(out_probes) print "[RUNNING: chr%d] Calculating ZRPKM scores..." % chr rpkm = np.apply_along_axis(cf.zrpkm, 0, rpkm, median[probe_mask], sd[probe_mask]) # svd transform print "[RUNNING: chr%d] SVD decomposition..." % chr components_removed = int(args.svd) U, S, Vt = np.linalg.svd(rpkm,full_matrices=False) new_S = np.diag(np.hstack([np.zeros([components_removed]),S[components_removed:]])) if args.write_svals != "": sval_f.write('chr' + str(chr) + '\t' + '\t'.join([str(_i) for _i in S]) + "\n") if args.plot_scree != "": ax.plot(S, label='chr' + str(chr),lw=0.5) # reconstruct data matrix rpkm = np.dot(U, np.dot(new_S, Vt)) # save to HDF5 file print "[RUNNING: chr%d] Saving SVD-ZRPKM values" % chr for i,s in enumerate(samples): out_data = np.empty(num_chr_probes,dtype='u4,f8') out_data['f0'] = probeIDs out_data['f1'] = rpkm[:,i] sample_tbl = h5file_out.createTable(chr_group,"sample_" + str(s),cf.rpkm_value,"%s" % str(s)) sample_tbl.append(out_data) print "[RUNNING] Saving sampleIDs to file..." sample_group = h5file_out.createGroup("/","samples","samples") sample_table = h5file_out.createTable(sample_group,"samples",cf.sample,"samples") dt = np.dtype([('sampleID',np.str_,100)]) out_samples = np.empty(len(samples.keys()),dtype=dt) out_samples['sampleID'] = np.array(samples.keys()) sample_table.append(out_samples) if args.write_sd != "": print "[RUNNING] Calculating standard deviations for all samples (this can take a while)..." sd_file = open(args.write_sd,'w') for i,s in enumerate(samples): # collect all SVD-ZRPKM values count = 1 for chr in chrs_to_process: if count == 1: sd_out = h5file_out.root._f_getChild("chr%d" % chr)._f_getChild("sample_%s" % s).read(field="rpkm").flatten() else: sd_out = np.hstack([sd_out,out.h5file_out.root._f_getChild("chr%d" % chr)._f_getChild("sample_%s" % s).read(field="rpkm").flatten()]) sd = np.std(sd_out) sd_file.write("%s\t%f\n" % (s,sd)) sd_file.close() if args.plot_scree != "": plt.title("Scree plot") if len(samples) < 50: plt.xlim([0,len(samples)]) plt.xlabel("S values") else: plt.xlim([0,50]) plt.xlabel("S values (only first 50 plotted)") plt.ylabel("Relative contributed variance") plt.savefig(args.plot_scree) print "[FINISHED]" h5file_out.close() sys.exit(0)