def make_matrix(samples, callers, annot_header, annot_data, named_data, call_data): # annot_header list of headers for annot_data. # annot_data list of tuples: chrom, pos, ref, alt[, more] # named_data list of (name, headers, all_annots) # call_data list of tuples: chrom, pos, ref, alt, sample, caller, call # chrom string # pos int # ref string # alt string # sample string # caller string # call Call object from genomicode import AnnotationMatrix # Make sure there's no duplicates. assert annot_header[:4] == ["Chrom", "Pos", "Ref", "Alt"] seen = {} for x in annot_data: x = x[:4] x = tuple(x) assert x not in seen, "Duplicate" seen[x] = 1 # Make annotation matrix. for x in annot_data: assert len(x) == len(annot_header) headers = annot_header all_annots = [] for i in range(len(headers)): x = [x[i] for x in annot_data] all_annots.append(x) annot_matrix = AnnotationMatrix.create_from_annotations( headers, all_annots) # Make named matrices. named_matrices = [] for x in named_data: name, headers, all_annots = x matrix = AnnotationMatrix.create_from_annotations(headers, all_annots) x = name, matrix named_matrices.append(x) # Make call matrix. call_matrix = SparseCallMatrix(call_data) return SimpleVariantMatrix(samples, callers, annot_matrix, named_matrices, call_matrix)
def add_snpeff_to_svm(svm_file, snpeff_file, outfile): import shutil from genomicode import filelib from genomicode import SimpleVariantMatrix from genomicode import AnnotationMatrix if not filelib.exists_nz(snpeff_file): shutil.copy2(svm_file, outfile) return # Read the annotations. header = None # includes Chrom, Pos, Ref, Alt coord2d = {} for d in filelib.read_row(snpeff_file, header=1): if header is None: header = d._header coord = d.Chrom, d.Pos, d.Ref, d.Alt coord2d[coord] = d svm = SimpleVariantMatrix.read_as_am(svm_file) CHROM = svm.header2annots["______Chrom"] POS = svm.header2annots["______Pos"] REF = svm.header2annots["______Ref"] ALT = svm.header2annots["______Alt"] snpeff_header = header[4:] snpeff_matrix = [] # Row major. for i in range(len(CHROM)): coord = CHROM[i], POS[i], REF[i], ALT[i] row = [""] * len(snpeff_header) d = coord2d.get(coord) if d: row = d._cols[4:] assert len(row) == len(snpeff_header) snpeff_matrix.append(row) assert len(snpeff_matrix) == len(CHROM) # AnnotationMatrix is column major. snpeff_annots = [] for j in range(len(snpeff_header)): x = [snpeff_matrix[i][j] for i in range(len(snpeff_matrix))] snpeff_annots.append(x) # Convert the headers to SVM format. snpeff_header = ["SnpEff______%s" % x for x in snpeff_header] # Make the new SimpleVariantMatrix. headers = svm.headers[:4] + snpeff_header + svm.headers[4:] x = [svm.header2annots[x] for x in svm.headers_h] all_annots = x[:4] + snpeff_annots + x[4:] merged = AnnotationMatrix.create_from_annotations( headers, all_annots, headerlines=svm.headerlines) SimpleVariantMatrix.write_from_am(outfile, merged)
def annotate_linked_variants(MATRIX, args): if not args: return MATRIX from genomicode import filelib from genomicode import AnnotationMatrix link_file = args filelib.assert_exists_nz(link_file) coord2perc = {} for d in filelib.read_row(link_file, header=1): chrom = d.Chrom pos = int(d.Pos) perc = float(d.Perc_Linked) coord2perc[(chrom, pos)] = perc chrom = MATRIX.header2annots["______Chrom"] pos = MATRIX.header2annots["______Pos"] pos = [int(x) for x in pos] link_score = [""] * len(chrom) for i in range(len(chrom)): link_score[i] = coord2perc.get((chrom[i], pos[i]), "") # Add after: # Chrom, Pos, Ref, Alt header = "Linkage______Score" assert header not in MATRIX.headers headers = MATRIX.headers[:4] + [header] + MATRIX.headers[4:] all_annots = [] for h in headers: if h != header: x = MATRIX[h] else: x = link_score all_annots.append(x) return AnnotationMatrix.create_from_annotations(headers, all_annots, MATRIX.headerlines)
def run(self, network, in_data, out_attributes, user_options, num_cores, outfile): import math from genomicode import filelib from genomicode import jmath from genomicode import AnnotationMatrix from genomicode import SimpleVariantMatrix from Betsy import module_utils as mlib svm_node = in_data filelib.assert_exists_nz(svm_node.identifier) linked_file = mlib.get_user_option(user_options, "linked_variants_file", not_empty=True, check_file=True) # Read the variant file. SVM = SimpleVariantMatrix.read_as_am(svm_node.identifier) CHROM = SVM["______Chrom"] POS = SVM["______Pos"] POS = [int(x) for x in POS] all_coords = {} # (chrom, pos) -> 1 for x in zip(CHROM, POS): all_coords[x] = 1 # Read the linked variant file. # Chrom Pos Perc Linked p coord2info = {} # (chrom, pos) -> d for d in filelib.read_row(linked_file, header=1): pos = int(d.Pos) if (d.Chrom, pos) not in all_coords: continue coord2info[(d.Chrom, pos)] = d # Align the linked annotations to the matrix. MAX_SCORE = 1000 min_p = 10**-(MAX_SCORE / 10) linked_headers = ["Perc Linked", "Score"] annotations = [] for (chrom, pos) in zip(CHROM, POS): if (chrom, pos) not in coord2info: x = [""] * len(linked_headers) annotations.append(x) continue d = coord2info[(chrom, pos)] score = MAX_SCORE if float(d.p) >= min_p: score = -10 * math.log(float(d.p), 10) x = d.Perc_Linked, score assert len(x) == len(linked_headers) annotations.append(x) # Convert the headers and annotations to SVM format. linked_headers = ["Linkage______%s" % x for x in linked_headers] linked_annotations = jmath.transpose(annotations) # Make the new SimpleVariantMatrix. # Figure out where to put these annotations. INDEX = 4 ## If Annovar exists, put after. #I = [i for (i, x) in enumerate(SVM.headers) # if x.upper().startswith("ANNOVAR")] #if I: # INDEX = max(INDEX, max(I)+1) headers = SVM.headers[:INDEX] + linked_headers + SVM.headers[INDEX:] x = [SVM.header2annots[x] for x in SVM.headers_h] all_annots = x[:INDEX] + linked_annotations + x[INDEX:] merged = AnnotationMatrix.create_from_annotations( headers, all_annots, headerlines=SVM.headerlines) SimpleVariantMatrix.write_from_am(outfile, merged)
def run( self, network, in_data, out_attributes, user_options, num_cores, outfile): from genomicode import filelib from genomicode import hashlib from genomicode import jmath from genomicode import AnnotationMatrix from genomicode import SimpleVariantMatrix from Betsy import module_utils as mlib simple_node = in_data filelib.assert_exists_nz(simple_node.identifier) gene_file = mlib.get_user_option( user_options, "cancer_genes_file", not_empty=True, check_file=True) # Read the cancer genes file. # <Gene ID> <Gene Symbol> <Dataset> ... symbol2info = {} # symbol -> d gene_iter = filelib.read_row(gene_file, header=1) header = None for d in gene_iter: assert "Gene Symbol" in d._header if header is None: header = [ x for x in d._header if x not in ["Gene ID", "Gene Symbol"]] if not d.Gene_Symbol: continue symbol2info[d.Gene_Symbol] = d # Read the variant file. SVM = SimpleVariantMatrix.read_as_am(simple_node.identifier) GENE_H = "Annovar______Gene.refGene" assert GENE_H in SVM.headers, "Missing annotation: %s" % GENE_H GENES = SVM[GENE_H] # Align the matrix to the simple variant matrix. gene_headers = header gene_annotations = [] for i, gene_str in enumerate(GENES): # Format of genes: # PFN1P2 # PMS2P2,PMS2P7 values = [""] * len(gene_headers) genes = gene_str.split(",") for gene in genes: if gene not in symbol2info: continue d = symbol2info[gene] for j, h in enumerate(gene_headers): h = hashlib.hash_var(h) assert hasattr(d, h) x = getattr(d, h) assert x in ["", "1"] if x == "1": values[j] = 1 gene_annotations.append(values) # Convert the headers and annotations to SVM format. gene_headers = ["Cancer Genes______%s" % x for x in gene_headers] gene_annotations = jmath.transpose(gene_annotations) # Make the new SimpleVariantMatrix. # Figure out where to put these annotations. INDEX = 4 # If Annovar exists, put after. I = [i for (i, x) in enumerate(SVM.headers) if x.upper().startswith("ANNOVAR")] if I: INDEX = max(INDEX, max(I)+1) # If SnpEff exists, put after. I = [i for (i, x) in enumerate(SVM.headers) if x.upper().startswith("SNPEFF")] if I: INDEX = max(INDEX, max(I)+1) # If COSMIC exists, put after. I = [i for (i, x) in enumerate(SVM.headers) if x.upper().startswith("COSMIC")] if I: INDEX = max(INDEX, max(I)+1) headers = SVM.headers[:INDEX] + gene_headers + SVM.headers[INDEX:] x = [SVM.header2annots[x] for x in SVM.headers_h] all_annots = x[:INDEX] + gene_annotations + x[INDEX:] merged = AnnotationMatrix.create_from_annotations( headers, all_annots, headerlines=SVM.headerlines) SimpleVariantMatrix.write_from_am(outfile, merged)
def run(self, network, in_data, out_attributes, user_options, num_cores, out_filename): #import shutil from genomicode import filelib from genomicode import parallel from genomicode import alignlib from genomicode import SimpleVariantMatrix from genomicode import AnnotationMatrix from Betsy import module_utils as mlib summary_node = in_data summary_filename = summary_node.identifier metadata = {} buildver = mlib.get_user_option(user_options, "annovar_buildver", allowed_values=["hg19"], not_empty=True) # Name files. p, root, ext = mlib.splitpath(summary_filename) annovar_infile = "pos.txt" log_filename = "annovar.log" # Annovar takes a filestem, without the ".vcf". annovar_outstem = "annotations" # Produces file: # <annovar_outstem>.hg19_multianno.txt multianno_file = "%s.hg19_multianno.txt" % annovar_outstem #temp_file = "temp.txt" # Make the infile for Annovar. # <chrom> <start> <end> <ref> <alt> handle = open(annovar_infile, 'w') for d in filelib.read_row(summary_filename, skip=2, header=1): x = d.Chrom, d.Pos, d.Pos, d.Ref, d.Alt print >> handle, "\t".join(x) handle.close() cmd = alignlib.make_annovar_command(annovar_infile, log_filename, annovar_outstem, buildver, vcf_input=False) parallel.sshell(cmd) metadata["commands"] = [cmd] filelib.assert_exists_nz(log_filename) filelib.assert_exists_nz(multianno_file) matrix = SimpleVariantMatrix.read(summary_filename) annot_matrix = matrix.annot_matrix #headers = annot_matrix.headers + anno_header[5:] chrom, pos = annot_matrix["Chrom"], annot_matrix["Pos"] ref, alt = annot_matrix["Ref"], annot_matrix["Alt"] pos = [int(x) for x in pos] # Read in the multianno output file. pos2d = {} # (chrom, start, ref, alt) -> d anno_header = None for d in filelib.read_row(multianno_file, header=1): key = d.Chr, int(d.Start), d.Ref, d.Alt assert key not in pos2d, "Duplicate pos: %s" % str(key) pos2d[key] = d if not anno_header: anno_header = d._header assert anno_header # Multianno starts with: # Chr Start End Ref Alt # Ignore these. assert anno_header[:5] == ["Chr", "Start", "End", "Ref", "Alt"] headers = anno_header[5:] all_annots = [] #for h in annot_matrix.headers_h: # x = annot_matrix.header2annots[h] # all_annots.append(x) for i in range(5, len(anno_header)): annots = [] for coord in zip(chrom, pos, ref, alt): d = pos2d.get(coord) x = "" if d: x = d._cols[i] annots.append(x) all_annots.append(annots) x = AnnotationMatrix.create_from_annotations(headers, all_annots) matrix.named_matrices.insert(0, ("Annovar", x)) SimpleVariantMatrix.write(out_filename, matrix) ## cols_to_add = len(anno_header) - 5 ## assert cols_to_add > 0 ## # Merge the multianno file with the simple call summary. Add ## # these columns before the <Sample>. ## # Sample <Sample> ## # Caller <Caller> ## # Chrom Pos Ref Alt Ref/Alt/VAF ## handle = open(temp_file, 'w') ## it = filelib.read_cols(summary_filename) ## header1 = it.next() ## header2 = it.next() ## header3 = it.next() ## assert len(header1) == len(header2), "%d %d %d %s" % ( ## len(header1), len(header2), len(header3), summary_filename) ## assert len(header1) == len(header3), "%d %d %d %s" % ( ## len(header1), len(header2), len(header3), summary_filename) ## assert header1[0] == "Sample" ## assert header2[0] == "Caller" ## assert header3[:4] == ["Chrom", "Pos", "Ref", "Alt"] ## header1 = header1[:4] + [""]*cols_to_add + header1[4:] ## header2 = header2[:4] + [""]*cols_to_add + header2[4:] ## header3 = header3[:4] + anno_header[5:] + header3[4:] ## print >>handle, "\t".join(header1) ## print >>handle, "\t".join(header2) ## print >>handle, "\t".join(header3) ## for cols in it: ## chrom, pos, ref, alt = cols[:4] ## pos = int(pos) ## d = pos2d.get((chrom, pos)) ## if not d: ## cols = cols[:4] + [""]*cols_to_add + cols[4:] ## continue ## assert ref == d.Ref, "%s %s %s %s %s %s" % ( ## chrom, pos, ref, alt, d.Ref, d.Alt) ## assert alt == d.Alt, "%s %s %s %s %s %s" % ( ## chrom, pos, ref, alt, d.Ref, d.Alt) ## x = d._cols[5:] ## assert len(x) == cols_to_add ## cols = cols[:4] + x + cols[4:] ## print >>handle, "\t".join(cols) ## handle.close() ## shutil.move(temp_file, out_filename) return metadata
def run( self, network, antecedents, out_attributes, user_options, num_cores, out_filename): import arrayio from genomicode import filelib from genomicode import AnnotationMatrix from genomicode import SimpleVariantMatrix simple_node, signal_node = antecedents filelib.assert_exists_nz(simple_node.identifier) filelib.assert_exists_nz(signal_node.identifier) # Read the variant file. SVM = SimpleVariantMatrix.read(simple_node.identifier) #AM = SVM.annot_matrix #assert GENE_H in AM.headers # Read the gene expression file. GXP = arrayio.read(signal_node.identifier) # Make sure the samples from the variant matrix can be found # in the gene expression matrix. GXP_samples = GXP.col_names(arrayio.COL_ID) missing = [x for x in SVM.samples if x not in GXP_samples] assert len(missing) < len(SVM.samples), ( "SimpleVariantMatrix and gene expression file have " "no common samples.") # Actually, may not have all the same samples. For example, a # gene expression profile might not have been calculated for # the germline sample. So ignore if something is missing. #x = missing #if len(x) > 5: # x = x[:5] + ["..."] #msg = "Samples (%d) not found in gene expression file: %s" % ( # len(missing), ", ".join(x)) #assert not missing, msg # Add all the samples from the gene expression file. SAMPLES = GXP_samples # Find the genes in each row. GENE_H = "Gene.refGene" annovar_matrix = None for (name, matrix) in SVM.named_matrices: if GENE_H in matrix.headers: annovar_matrix = matrix break assert annovar_matrix, "Missing annotation: %s" % GENE_H GENES = annovar_matrix[GENE_H] # Make a list of the genes. genes = {} for i, gene_str in enumerate(GENES): # Format of genes: # PFN1P2 # PMS2P2,PMS2P7 for x in gene_str.split(","): genes[x] = 1 genes = sorted(genes) # Make a matrix of the gene expression values for each gene # and each sample. #I = [GXP_samples.index(x) for x in SVM.samples] #GXP_a = GXP.matrix(genes, I) # align the matrices. GXP_a = GXP.matrix(genes, None) # Write out the expression matrix for debugging purposes. arrayio.write(GXP_a, "expression.txt") # Search for each of the genes in the matrix. gene2I = {} # gene -> list of row indexes for gene in genes: x = GXP_a._index(row=gene) I_row, i_col = x if I_row: gene2I[gene] = I_row # Align the gene expression matrix to the simple variant # matrix. #matrix = [[None]*len(SVM.samples) for i in range(len(GENES))] matrix = [[None]*len(SAMPLES) for i in range(len(GENES))] for i, gene_str in enumerate(GENES): # Format of genes: Format of output # PFN1P2 5.2 # PMS2P2,PMS2P7 2.2,8.6 # If a gene is missing, then skip it. genes = gene_str.split(",") #for j in range(len(SVM.samples)): for j in range(len(SAMPLES)): values = [] # expression values for each gene. for k in range(len(genes)): if genes[k] not in gene2I: continue x = [GXP_a._X[l][j] for l in gene2I[genes[k]]] # If there are multiple instances of this gene, # then pick the one with the maximum expression. x = max(x) values.append(x) values = [_pretty_gxp(x) for x in values] x = ",".join(values) matrix[i][j] = x # Add the matrix back to the simple variant matrix. #headers = SVM.samples headers = SAMPLES all_annots = [] for j in range(len(headers)): x = [matrix[i][j] for i in range(len(matrix))] all_annots.append(x) x = AnnotationMatrix.create_from_annotations(headers, all_annots) SVM.named_matrices.append(("Gene Expression", x)) # Write to file. SimpleVariantMatrix.write(out_filename, SVM)
def add_coverage_to_svm(svm_file, coverage_file, outfile, is_rna_cov): from genomicode import jmath from genomicode import filelib from genomicode import AnnotationMatrix from genomicode import SimpleVariantMatrix # Read the variant file. SVM = SimpleVariantMatrix.read(svm_file) AM = SVM.annot_matrix assert "Chrom" in AM.headers assert "Pos" in AM.headers CHROM = AM["Chrom"] POS = AM["Pos"] POS = [int(x) for x in POS] # Read the coverage matrix. # Chrom Pos <Sample> [<Sample> ...] # Pos is 1-based. coord2sample2cov = {} # (chrom, pos) -> sample -> ref/alt/vaf cov_samples = {} for d in filelib.read_row(coverage_file, header=1): coord = d.Chrom, int(d.Pos) if coord not in coord2sample2cov: coord2sample2cov[coord] = {} for i in range(2, len(d._header)): sample = d._header[i] cov = d._cols[i] if not cov: continue #coord2sample2cov[coord][sample] = int(cov) coord2sample2cov[coord][sample] = cov cov_samples[sample] = 1 # Make sure the samples from the variant matrix can be found # in the coverage matrix. missing = [x for x in SVM.samples if x not in cov_samples] assert len(missing) < len(SVM.samples), ( "SimpleVariantMatrix and coverage file have " "no common samples.") # If the samples aren't sequenced at high coverage, it's # possible they just don't have reads at these positions. Be # a little lenient here, and accept the file if some of the # samples overlap. #x = missing #if len(x) > 5: # x = x[:5] + ["..."] #msg = "Samples (%d) not found in coverage file: %s" % ( # len(missing), ", ".join(x)) #assert not missing, msg # Report the coverage for the samples at the intersection. SAMPLES = [x for x in SVM.samples if x in cov_samples] # Align the matrix to the simple variant matrix. #matrix = [[None]*len(SVM.samples) for i in range(AM.num_annots())] matrix = [[None]*len(SAMPLES) for i in range(AM.num_annots())] for i in range(AM.num_annots()): coord = CHROM[i], POS[i] sample2cov = coord2sample2cov.get(coord, {}) x = [sample2cov.get(x, "") for x in SAMPLES] #x = map(str, x) matrix[i] = x # Add the matrix back to the simple variant matrix. headers = SAMPLES all_annots = jmath.transpose(matrix) name = "Coverage" # If this is being used to add RNA coverage, use a different # name. if is_rna_cov: name = "RNA Coverage" x = AnnotationMatrix.create_from_annotations(headers, all_annots) SVM.named_matrices.append((name, x)) # Write to file. SimpleVariantMatrix.write(outfile, SVM)
def run( self, network, in_data, out_attributes, user_options, num_cores, outfile): from genomicode import filelib from genomicode import jmath from genomicode import AnnotationMatrix from genomicode import SimpleVariantMatrix from Betsy import module_utils as mlib svm_node = in_data filelib.assert_exists_nz(svm_node.identifier) cosmic_file = mlib.get_user_option( user_options, "cosmic_variants_file", not_empty=True, check_file=True) # Read the variant file. SVM = SimpleVariantMatrix.read_as_am(svm_node.identifier) CHROM = SVM["______Chrom"] POS = SVM["______Pos"] POS = [int(x) for x in POS] all_coords = {} # (chrom, pos) -> 1 for x in zip(CHROM, POS): all_coords[x] = 1 # Read the COSMIC variant file. # Chrom Start End GRCh Count SNP # Mutation CDS Mutation AA # FATHMM prediction FATHMM score Mutation somatic status coord2info = {} # (chrom, pos) -> d for d in filelib.read_row(cosmic_file, header=1): start, end = int(d.Start), int(d.End) in_svm = False for pos in range(start, end+1): if (d.Chrom, pos) in all_coords: in_svm = True break if not in_svm: continue coord2info[(d.Chrom, pos)] = d # Align the COSMIC annotations to the matrix. cosmic_headers = [ "SNP", "Num Tumors", "Mutation CDS", "Mutation AA", "FATHMM prediction", "FATHMM score", "Mutation somatic status"] annotations = [] for (chrom, pos) in zip(CHROM, POS): if (chrom, pos) not in coord2info: x = [""] * len(cosmic_headers) annotations.append(x) continue d = coord2info[(chrom, pos)] x = d.SNP, d.Count, d.Mutation_CDS, d.Mutation_AA, \ d.FATHMM_prediction, d.FATHMM_score, \ d.Mutation_somatic_status annotations.append(x) # Convert the headers and annotations to SVM format. cosmic_headers = ["COSMIC______%s" % x for x in cosmic_headers] cosmic_annotations = jmath.transpose(annotations) # Make the new SimpleVariantMatrix. # Figure out where to put these annotations. INDEX = 4 # If Annovar exists, put after. I = [i for (i, x) in enumerate(SVM.headers) if x.upper().startswith("ANNOVAR")] if I: INDEX = max(INDEX, max(I)+1) # If SnpEff exists, put after. I = [i for (i, x) in enumerate(SVM.headers) if x.upper().startswith("SNPEFF")] if I: INDEX = max(INDEX, max(I)+1) headers = SVM.headers[:INDEX] + cosmic_headers + SVM.headers[INDEX:] x = [SVM.header2annots[x] for x in SVM.headers_h] all_annots = x[:INDEX] + cosmic_annotations + x[INDEX:] merged = AnnotationMatrix.create_from_annotations( headers, all_annots, headerlines=SVM.headerlines) SimpleVariantMatrix.write_from_am(outfile, merged)
def read_as_am(filename, is_csv=False): # Read file in SVM format. Return an AnnotationMatrix object. # Does no special processing on any columns (i.e. no parsing as # integers or Call objects). Everything is a string. # Header format: <header0>___<header1>___<header2> # "blanks" are filled in. E.g. "Annovar" occurs in each Annovar # column in header0. # # Headers: # ______Chrom # ______Pos # ______Ref # ______Alt # Num Callers______<Sample> # ... from genomicode import filelib from genomicode import AnnotationMatrix delimiter = "\t" if is_csv: delimiter = "," matrix = [] for x in filelib.read_cols(filename, delimiter=delimiter): matrix.append(x) assert len(matrix) >= 3 # at least 3 rows for the header for i in range(1, len(matrix)): assert len(matrix[i]) == len(matrix[0]) assert len(matrix[0]) >= 4 # Chrom, Pos, Ref, Alt assert len(matrix[0]) >= 5, "No calls" header0 = matrix[0] header1 = matrix[1] header2 = matrix[2] assert header2[:4] == ["Chrom", "Pos", "Ref", "Alt"] # Fill in the blanks for header1. for i in range(1, len(header1)): if header1[i]: continue # header1[i] is blank. If header0[i], then this starts a new # "block". Start with a new header1, and do not copy the old # one over. if not header1[i] and not header0[i]: header1[i] = header1[i - 1] # Fill in the blanks for header0. for i in range(1, len(header0)): if not header0[i]: header0[i] = header0[i - 1] # Make a list of all samples. I = [i for (i, x) in enumerate(header2) if x == "Ref/Alt/VAF"] assert I x = [header0[i] for i in I] x = [x for x in x if x] # Get rid of duplicates, preserving order. x = [x[i] for (i, y) in enumerate(x) if y not in x[:i]] samples = x # Make a list of all callers. x = [header1[i] for i in I] x = [x for x in x if x] # Get rid of duplicates, preserving order. x = [x[i] for (i, y) in enumerate(x) if y not in x[:i]] callers = x headers = [] for x in zip(header0, header1, header2): x = "___".join(x) headers.append(x) all_annots = [] for j in range(len(headers)): annots = [x[j] for x in matrix[3:]] all_annots.append(annots) matrix = AnnotationMatrix.create_from_annotations(headers, all_annots) matrix.samples = samples matrix.callers = callers return matrix
def run(self, network, in_data, out_attributes, user_options, num_cores, out_filename): from genomicode import filelib from genomicode import SimpleVariantMatrix from genomicode import AnnotationMatrix simple_file = in_data.identifier metadata = {} # Read all in memory. Hopefully, not too big. ds = [] for d in filelib.read_row(simple_file, header=-1): ds.append(d) #if len(ds) > 50000: # DEBUG # break # MuSE sometimes has alternates. # Alt A,C # Num_Alt 13,0 # VAF 0.19,0.0 # Detect this and fix it. Take the alternate with the highest VAF. for d in ds: if d.Num_Alt.find(",") < 0: continue x1 = d.Num_Alt.split(",") x2 = d.VAF.split(",") assert len(x1) == len(x2) x1 = map(int, x1) x2 = map(float, x2) max_vaf = max_i = None for i in range(len(x2)): if max_vaf is None or x2[i] > max_vaf: max_vaf = x2[i] max_i = i assert max_i is not None d.Num_Alt = str(x1[max_i]) d.VAF = str(x2[max_i]) # Make a list of all the positions. positions = {} # (Chrom, Pos) -> 1 for d in ds: positions[(d.Chrom, int(d.Pos))] = 1 positions = sorted(positions) # Make a list of all the callers. callers = {} for d in ds: callers[d.Caller] = 1 callers = sorted(callers) # Make a list of all the samples. samples = {} for d in ds: samples[d.Sample] = 1 samples = sorted(samples) # Make a list of the coordinates. coord_data = {} for d in ds: x = d.Chrom, int(d.Pos), d.Ref, d.Alt coord_data[x] = 1 coord_data = sorted(coord_data) # Make a list of all DNA calls. call_data = [] for d in ds: assert d.Source in ["DNA", "RNA"] if d.Source != "DNA": continue num_ref = num_alt = vaf = None if d.Num_Ref: num_ref = int(d.Num_Ref) if d.Num_Alt: num_alt = int(d.Num_Alt) if d.VAF: vaf = float(d.VAF) if num_ref is None and num_alt is None and vaf is None: continue call = SimpleVariantMatrix.Call(num_ref, num_alt, vaf) x = d.Chrom, int(d.Pos), d.Ref, d.Alt, d.Sample, d.Caller, call call_data.append(x) # sample -> caller -> chrom, pos, ref, alt -> call samp2caller2coord2call = {} for x in call_data: chrom, pos, ref, alt, sample, caller, call = x coord = chrom, pos, ref, alt if sample not in samp2caller2coord2call: samp2caller2coord2call[sample] = {} caller2coord2call = samp2caller2coord2call[sample] if caller not in caller2coord2call: caller2coord2call[caller] = {} coord2call = caller2coord2call[caller] # A (sample, caller, coord) may have multiple calls. For # example, for germline samples that are called with each # tumor sample. If this is the case, then take the call # with the highest coverage. if coord in coord2call: old_call = coord2call[coord] cov = old_cov = None if call.num_ref is not None and call.num_alt is not None: cov = call.num_ref + call.num_alt if old_call.num_ref is not None and \ old_call.num_alt is not None: old_cov = old_call.num_ref + old_call.num_alt if cov is None and old_cov is not None: call = old_call elif cov is not None and old_cov is not None and cov < old_cov: call = old_call coord2call[coord] = call # Count the number of callers that called a variant at each # position for each sample. samp2coord2caller = {} # sample -> chrom, pos, ref, alt -> caller -> 1 # Need to do this first, to make sure each caller is counted # at most once. This is to account for germline samples that # is called by each caller multiple times. for x in call_data: chrom, pos, ref, alt, sample, caller, call = x coord = chrom, pos, ref, alt if sample not in samp2coord2caller: samp2coord2caller[sample] = {} if coord not in samp2coord2caller[sample]: samp2coord2caller[sample][coord] = {} samp2coord2caller[sample][coord][caller] = 1 samp2coord2nc = {} # sample -> chrom, pos, ref, alt -> num_callers for sample in samp2coord2caller: samp2coord2nc[sample] = {} for coord in samp2coord2caller[sample]: samp2coord2nc[sample][coord] = len( samp2coord2caller[sample][coord]) #for x in call_data: # chrom, pos, ref, alt, sample, caller, call = x # coord = chrom, pos, ref, alt # if sample not in samp2coord2nc: # samp2coord2nc[sample] = {} # nc = samp2coord2nc[sample].get(coord, 0) + 1 # samp2coord2nc[sample][coord] = nc # Format everything into an annotation matrix. headers0 = [] headers1 = [] headers2 = [] all_annots = [] # Add the positions. headers0 += ["", "", "", ""] headers1 += ["", "", "", ""] headers2 += ["Chrom", "Pos", "Ref", "Alt"] for i in range(4): x = [x[i] for x in coord_data] x = [str(x) for x in x] all_annots.append(x) # Add the number of callers information. headers0 += ["Num Callers"] * len(samples) headers1 += [""] * len(samples) headers2 += samples for sample in samples: annots = [] for coord in coord_data: nc = samp2coord2nc.get(sample, {}).get(coord, "") annots.append(nc) all_annots.append(annots) # Add information about calls. for sample in samples: caller2coord2call = samp2caller2coord2call.get(sample, {}) for i, caller in enumerate(callers): h0 = "" if not i: h0 = sample h1 = caller h2 = "Ref/Alt/VAF" headers0.append(h0) headers1.append(h1) headers2.append(h2) coord2call = caller2coord2call.get(caller, {}) annots = [] for coord in coord_data: x = "" call = coord2call.get(coord) if call: x = SimpleVariantMatrix._format_call(call) annots.append(x) all_annots.append(annots) # Set the headers. assert len(headers0) == len(headers1) assert len(headers0) == len(headers2) assert len(headers0) == len(all_annots) headers = [None] * len(headers0) for i, x in enumerate(zip(headers0, headers1, headers2)): x = "___".join(x) headers[i] = x matrix = AnnotationMatrix.create_from_annotations(headers, all_annots) SimpleVariantMatrix.write_from_am(out_filename, matrix) #annot_header = ["Chrom", "Pos", "Ref", "Alt"] #matrix = SimpleVariantMatrix.make_matrix( # samples, callers, annot_header, coord_data, named_data, # call_data) #SimpleVariantMatrix.write(out_filename, matrix) return metadata