def matrix_merge(matrices, reducer=merge_require_equal): """ Merge several matrices. Matrices should have the same row names. Where multiple matrices have the same column, the values are merged using the reducer function. """ columns = collections.OrderedDict() row_names = matrices[0].keys() for i, matrix in enumerate(matrices): assert row_names == matrix.keys( ), 'Row names don\'t match in matrix merge' for column in matrix.value_type().keys(): if column not in columns: columns[column] = [] columns[column].append(i) return io.named_matrix_type(row_names, columns.keys())([[ reduce(reducer, [matrices[i][name][column] for i in columns[column]]) for column in columns ] for name in row_names])
def matrix_merge(matrices, reducer=merge_require_equal): """ Merge several matrices. Matrices should have the same row names. Where multiple matrices have the same column, the values are merged using the reducer function. """ columns = collections.OrderedDict() row_names = matrices[0].keys() for i, matrix in enumerate(matrices): assert row_names == matrix.keys(), 'Row names don\'t match in matrix merge' for column in matrix.value_type().keys(): if column not in columns: columns[column] = [ ] columns[column].append( i ) return io.named_matrix_type(row_names, columns.keys())([ [ reduce(reducer, [ matrices[i][name][column] for i in columns[column] ]) for column in columns ] for name in row_names ])
def run(self): #assert not self.utr_only or self.utrs, '--utrs-only yes but no --utrs given' # Reference genome #chromosome_lengths = reference_directory.Reference(self.reference, must_exist=True).get_lengths() chromosomes = collections.OrderedDict(io.read_sequences(self.reference)) def get_interpeak_seq(peaks): start = min(item.transcription_stop for item in peaks) end = max(item.transcription_stop for item in peaks) if end-start > self.max_seq: return '' if peaks[0].strand >= 0: return chromosomes[peaks[0].seqid][start:end] else: return bio.reverse_complement(chromosomes[peaks[0].seqid][start:end]) def get_prepeak_seq(gene,peaks): if gene.strand >= 0: start = gene.utr_pos end = min(item.transcription_stop for item in peaks) if end-start > self.max_seq: return '' return chromosomes[gene.seqid][start:end] else: start = max(item.transcription_stop for item in peaks) end = gene.utr_pos if end-start > self.max_seq: return '' return bio.reverse_complement(chromosomes[gene.seqid][start:end]) # Normalization files if self.norm_file: norm_file = self.norm_file else: nesoni.Norm_from_counts(self.prefix+'-norm', self.counts).run() norm_file = self.prefix+'-norm.csv' norms = io.read_grouped_table(norm_file, [('All',str)])['All'] pair_norm_names = [ ] pair_norms = [ ] for i in xrange(len(norms)): pair_norm_names.append(norms.keys()[i]+'-peak1') pair_norms.append(norms.values()[i]) for i in xrange(len(norms)): pair_norm_names.append(norms.keys()[i]+'-peak2') pair_norms.append(norms.values()[i]) io.write_grouped_csv( self.prefix+'-pairs-norm.csv', [('All',io.named_list_type(pair_norm_names)(pair_norms))], comments=['#Normalization'], ) # Read data annotations = list(annotation.read_annotations(self.parents)) if self.utrs: utrs = list(annotation.read_annotations(self.utrs)) else: utrs = [ ] children = list(annotation.read_annotations(self.children)) count_table = io.read_grouped_table(self.counts, [ ('Count',int), ('Tail_count',int), ('Tail',_float_or_none), ('Proportion',_float_or_none), ('Annotation',str) ]) counts = count_table['Count'] tail_counts = count_table['Tail_count'] proportions = count_table['Proportion'] tails = count_table['Tail'] samples = counts.value_type().keys() sample_tags = { } for line in count_table.comments: if line.startswith('#sampleTags='): parts = line[len('#sampleTags='):].split(',') assert parts[0] not in sample_tags sample_tags[parts[0]] = parts for item in children: item.weight = sum( counts[item.get_id()][name] * float(norms[name]['Normalizing.multiplier']) for name in samples ) parents = [ ] id_to_parent = { } for item in annotations: if item.type != self.parent_type: continue assert item.get_id() not in id_to_parent, 'Duplicate id in parent file: '+item.get_id() parents.append(item) id_to_parent[item.get_id()] = item item.children = [ ] #item.cds = [ ] # Default utr if item.strand >= 0: item.utr_pos = item.end else: item.utr_pos = item.start if 'three_prime_UTR_start' in item.attr: if item.strand >= 0: item.utr_pos = int(item.attr['three_prime_UTR_start'])-1 else: item.utr_pos = int(item.attr['three_prime_UTR_start']) for item in utrs: assert item.attr['Parent'] in id_to_parent, 'Unknown gene '+item.attr['Parent'] id_to_parent[item.attr['Parent']].utr_pos = (item.start if item.strand >= 0 else item.end) for item in children: item.transcription_stop = item.end if item.strand >= 0 else item.start #End of transcription, 0-based, ie between-positions based if 'Parent' in item.attr: for item_parent in item.attr['Parent'].split(','): parent = id_to_parent[item_parent] parent.children.append(item) for item in parents: item.children.sort(key=_annotation_sorter) relevant = list(item.children) if self.utr_only: #if item.strand <= 0: # relative_utr_start = item.end - int(item.attr['three_prime_UTR_start']) #else: # relative_utr_start = int(item.attr['three_prime_UTR_start'])-1 - item.start # #def relative_start(peak): # return item.end-peak.end if item.strand < 0 else peak.start-item.start #relevant = [ peak for peak in relevant if relative_start(peak) >= relative_utr_start ] relevant = [ peak for peak in relevant if (peak.end >= item.utr_pos if item.strand >= 0 else peak.start <= item.utr_pos) ] if self.top: relevant.sort(key=lambda peak:peak.weight, reverse=True) relevant = relevant[:self.top] relevant.sort(key=_annotation_sorter) item.relevant_children = relevant # JSON output j_data = { } j_genes = j_data['genes'] = { } j_genes['__comment__'] = 'start is 0-based' j_genes['name'] = [ ] j_genes['chromosome'] = [ ] j_genes['strand'] = [ ] j_genes['start'] = [ ] j_genes['utr'] = [ ] j_genes['end'] = [ ] j_genes['gene'] = [ ] j_genes['product'] = [ ] j_genes['peaks'] = [ ] j_genes['relevant_peaks'] = [ ] #j_genes['cds'] = [ ] #j_genes['cds_start'] = [ ] #j_genes['cds_end'] = [ ] for item in parents: j_genes['name'].append( item.get_id() ) j_genes['chromosome'].append( item.seqid ) j_genes['strand'].append( item.strand ) j_genes['start'].append( item.start ) j_genes['utr'].append( item.utr_pos ) j_genes['end'].append( item.end ) j_genes['gene'].append( item.attr.get('Name',item.attr.get('gene','')) ) j_genes['product'].append( item.attr.get('Product',item.attr.get('product','')) ) j_genes['peaks'].append( [ item2.get_id() for item2 in item.children ] ) j_genes['relevant_peaks'].append( [ item2.get_id() for item2 in item.relevant_children ] ) #j_genes['cds'].append( item.cds ) #j_genes['cds_start'].append( item.cds_start ) #j_genes['cds_end'].append( item.cds_end ) j_peaks = j_data['peaks'] = { } j_peaks['__comment__'] = 'start is 0-based' j_peaks['name'] = [ ] j_peaks['chromosome'] = [ ] j_peaks['strand'] = [ ] j_peaks['start'] = [ ] j_peaks['end'] = [ ] j_peaks['parents'] = [ ] j_peaks['counts'] = [ ] j_peaks['tail_lengths'] = [ ] j_peaks['proportion_tailed'] = [ ] for item in children: j_peaks['name'].append( item.get_id() ) j_peaks['chromosome'].append( item.seqid ) j_peaks['strand'].append( item.strand ) j_peaks['start'].append( item.start ) j_peaks['end'].append( item.end ) j_peaks['parents'].append( item.attr['Parent'].split(',') if 'Parent' in item.attr else [ ]) j_peaks['counts'].append( counts[item.get_id()].values() ) j_peaks['tail_lengths'].append( count_table['Tail'][item.get_id()].values() ) j_peaks['proportion_tailed'].append( count_table['Proportion'][item.get_id()].values() ) j_samples = j_data['samples'] = { } j_samples['name'] = [ ] j_samples['tags'] = [ ] j_samples['normalizing_multiplier'] = [ ] for name in samples: j_samples['name'].append(name) j_samples['tags'].append(sample_tags[name]) j_samples['normalizing_multiplier'].append(float(norms[name]['Normalizing.multiplier'])) j_chromosomes = j_data['chromosomes'] = { } j_chromosomes['name'] = [ ] j_chromosomes['length'] = [ ] for name, seq in chromosomes.iteritems(): j_chromosomes['name'].append(name) j_chromosomes['length'].append(len(seq)) with open(self.prefix + '.json','wb') as f: json.dump(j_data, f) # Output paired peak file output_comments = [ '#Counts' ] output_samples = [ ] for item in samples: output_samples.append(item+'-peak1') output_comments.append('#sampleTags=' + ','.join([item+'-peak1','peak1']+sample_tags.get(item,[]))) for item in samples: output_samples.append(item+'-peak2') output_comments.append('#sampleTags=' + ','.join([item+'-peak2','peak2']+sample_tags.get(item,[]))) output_names = [ ] output_counts = [ ] output_tail_counts = [ ] output_proportions = [ ] output_tails = [ ] output_annotation_fields = [ 'gene', 'product', 'mean_tail_1', 'mean_tail_2', 'chromosome', 'strand', 'transcription_stops' ] #, 'interpeak_seq', ] output_annotations = [ ] for item in parents: peaks = item.relevant_children for i in xrange(len(peaks)-1): for j in xrange(i+1, len(peaks)): id_i = peaks[i].get_id() id_j = peaks[j].get_id() id_pair = item.get_id() + '-'+id_i+'-'+id_j output_names.append(id_pair) row = [ ] row.extend(counts[id_i].values()) row.extend(counts[id_j].values()) output_counts.append(filter(_text,row)) row = [ ] row.extend(tail_counts[id_i].values()) row.extend(tail_counts[id_j].values()) output_tail_counts.append(filter(_text,row)) row = [ ] row.extend(proportions[id_i].values()) row.extend(proportions[id_j].values()) output_proportions.append(filter(_text,row)) row = [ ] row.extend(tails[id_i].values()) row.extend(tails[id_j].values()) output_tails.append(filter(_text,row)) output_annotations.append([ item.attr.get('Name',item.attr.get('gene','')), item.attr.get('Product',item.attr.get('product','')), count_table['Annotation'][id_i]['mean-tail'], count_table['Annotation'][id_j]['mean-tail'], item.seqid, str(item.strand), '%d, %d' % (peaks[i].transcription_stop,peaks[j].transcription_stop), #get_interpeak_seq([peaks[i],peaks[j]]), ]) #output_count_table = io.named_matrix_type(output_names,output_samples)(output_counts) io.write_grouped_csv( self.prefix + '-pairs.csv', [ ('Count',io.named_matrix_type(output_names,output_samples)(output_counts)), ('Tail_count',io.named_matrix_type(output_names,output_samples)(output_tail_counts)), ('Proportion',io.named_matrix_type(output_names,output_samples)(output_proportions)), ('Tail',io.named_matrix_type(output_names,output_samples)(output_tails)), ('Annotation',io.named_matrix_type(output_names,output_annotation_fields)(output_annotations)), ], comments=output_comments, ) # # Chi Sq tests # # #for id in relation: # # peaks = relation[id] # # if len(peaks) < 2: continue # # mats = [ ] # genes = [ ] # products = [ ] # mean_tails = [ ] # prop_tails = [ ] # # peak_names = [ ] # chromosome_names = [ ] # strands = [ ] # transcription_stops = [ ] # interpeak_seqs = [ ] # prepeak_seqs = [ ] # # for parent in parents: # id = parent.get_id() # peaks = parent.relevant_children # if len(peaks) < 2: continue # # matrix = [ ] # for item in peaks: # matrix.append(counts[item.get_id()].values()) # # mats.append( # runr.R_literal(id) + ' = ' + # runr.R_literal(matrix) # ) # # genes.append(parent.attr.get('Name',parent.attr.get('gene',''))) # products.append(parent.attr.get('Product',parent.attr.get('product',''))) # # def format_mean(s): # if s == 'NA': return 'NA' # return '%.1f' % float(s) # mean_tails.append(', '.join( format_mean(count_table['Annotation'][item.get_id()]['mean-tail']) for item in peaks )) # # def format_prop(s): # if s == 'NA': return 'NA' # return '%.2f' % float(s) # prop_tails.append(', '.join( format_prop(count_table['Annotation'][item.get_id()]['proportion-with-tail']) for item in peaks )) # # peak_names.append(', '.join(item.get_id() for item in peaks)) # chromosome_names.append(parent.seqid) # strands.append(parent.strand) # transcription_stops.append(', '.join(str(item.transcription_stop) for item in peaks)) # interpeak_seqs.append(get_interpeak_seq(peaks)) # prepeak_seqs.append(get_prepeak_seq(parent,peaks)) # # #if len(mats) >= 10: break # # text = 'cat("Loading data into R+\n")\n' # text += 'data <- list(\n' + ',\n'.join(mats) + ')\n' # text += CHISQ # # runr.run_script(text, # OUTPUT_FILENAME=self.prefix+'.csv', # GENES = genes, # PRODUCTS = products, # MEAN_TAILS = mean_tails, # PROP_TAILS = prop_tails, # PEAK_NAMES = peak_names, # CHROMOSOME_NAMES = chromosome_names, # STRANDS = strands, # TRANSCRIPTION_STOPS = transcription_stops, # INTERPEAK_SEQS = interpeak_seqs, # PREPEAK_SEQS = prepeak_seqs, # ) #
def run(self): data = io.read_grouped_table( self.counts, [('Count',str), ('Annotation',str), ('Tail_count',str), ('Tail',str), ('Proportion',str)], 'Count', ) features = data['Count'].keys() samples = data['Count'].value_type().keys() tags = { } for sample in samples: tags[sample] = [sample] for line in data.comments: if line.startswith('#sampleTags='): parts = line[len('#sampleTags='):].split(',') tags[parts[0]] = parts group_names = [ ] groups = [ ] group_tags = [ ] for item in self.groups: select = selection.term_specification(item) name = selection.term_name(item) group = [ item for item in samples if selection.matches(select, tags[item]) ] assert group, 'Empty group: '+name this_group_tags = [ name ] for tag in tags[group[0]]: if tag == name: continue for item in group[1:]: for item2 in tags[item]: if tag not in item2: break else: this_group_tags.append(tag) group_names.append(name) groups.append(group) group_tags.append(this_group_tags) result = io.Grouped_table() result.comments = [ '#Counts' ] for item in group_tags: result.comments.append('#sampleTags='+','.join(item)) count = [ ] tail_count = [ ] tail = [ ] proportion = [ ] for feature in features: this_count = [ ] this_tail_count = [ ] this_tail = [ ] this_proportion = [ ] for group in groups: this_this_count = [ ] this_this_tail_count = [ ] this_this_tail = [ ] this_this_proportion = [ ] for sample in group: this_this_count.append(int(data['Count'][feature][sample])) this_this_tail_count.append(int(data['Tail_count'][feature][sample])) item = data['Tail'][feature][sample] if item != 'NA': this_this_tail.append(float(item)) item = data['Proportion'][feature][sample] if item != 'NA': this_this_proportion.append(float(item)) this_count.append(str(sum(this_this_count))) this_tail_count.append(str(sum(this_this_tail_count))) this_tail.append(str(sum(this_this_tail)/len(this_this_tail)) if this_this_tail else 'NA') this_proportion.append(str(sum(this_this_proportion)/len(this_this_proportion)) if this_this_proportion else 'NA') count.append(this_count) tail_count.append(this_tail_count) tail.append(this_tail) proportion.append(this_proportion) matrix = io.named_matrix_type(features,group_names) result['Count'] = matrix(count) result['Annotation'] = data['Annotation'] result['Tail_count'] = matrix(tail_count) result['Tail'] = matrix(tail) result['Proportion'] = matrix(proportion) result.write_csv(self.prefix + '.csv')
def run(self): data = io.read_grouped_table( self.counts, [("Count", str), ("Annotation", str), ("Tail_count", str), ("Tail", str), ("Proportion", str)], "Count", ) features = data["Count"].keys() samples = data["Count"].value_type().keys() tags = {} for sample in samples: tags[sample] = [sample] for line in data.comments: if line.startswith("#sampleTags="): parts = line[len("#sampleTags=") :].split(",") tags[parts[0]] = parts group_names = [] groups = [] group_tags = [] for item in self.groups: select = selection.term_specification(item) name = selection.term_name(item) group = [item for item in samples if selection.matches(select, tags[item])] assert group, "Empty group: " + name this_group_tags = [name] for tag in tags[group[0]]: if tag == name: continue for item in group[1:]: for item2 in tags[item]: if tag not in item2: break else: this_group_tags.append(tag) group_names.append(name) groups.append(group) group_tags.append(this_group_tags) result = io.Grouped_table() result.comments = ["#Counts"] for item in group_tags: result.comments.append("#sampleTags=" + ",".join(item)) count = [] tail_count = [] tail = [] proportion = [] for feature in features: this_count = [] this_tail_count = [] this_tail = [] this_proportion = [] for group in groups: this_this_count = [] this_this_tail_count = [] this_this_tail = [] this_this_proportion = [] for sample in group: this_this_count.append(int(data["Count"][feature][sample])) this_this_tail_count.append(int(data["Tail_count"][feature][sample])) item = data["Tail"][feature][sample] if item != "NA": this_this_tail.append(float(item)) item = data["Proportion"][feature][sample] if item != "NA": this_this_proportion.append(float(item)) this_count.append(str(sum(this_this_count))) this_tail_count.append(str(sum(this_this_tail_count))) this_tail.append(str(sum(this_this_tail) / len(this_this_tail)) if this_this_tail else "NA") this_proportion.append( str(sum(this_this_proportion) / len(this_this_proportion)) if this_this_proportion else "NA" ) count.append(this_count) tail_count.append(this_tail_count) tail.append(this_tail) proportion.append(this_proportion) matrix = io.named_matrix_type(features, group_names) result["Count"] = matrix(count) result["Annotation"] = data["Annotation"] result["Tail_count"] = matrix(tail_count) result["Tail"] = matrix(tail) result["Proportion"] = matrix(proportion) result.write_csv(self.prefix + ".csv")
def run(self): #assert not self.utr_only or self.utrs, '--utrs-only yes but no --utrs given' # Reference genome #chromosome_lengths = reference_directory.Reference(self.reference, must_exist=True).get_lengths() chromosomes = collections.OrderedDict(io.read_sequences( self.reference)) def get_interpeak_seq(peaks): start = min(item.transcription_stop for item in peaks) end = max(item.transcription_stop for item in peaks) if end - start > self.max_seq: return '' if peaks[0].strand >= 0: return chromosomes[peaks[0].seqid][start:end] else: return bio.reverse_complement( chromosomes[peaks[0].seqid][start:end]) def get_prepeak_seq(gene, peaks): if gene.strand >= 0: start = gene.utr_pos end = min(item.transcription_stop for item in peaks) if end - start > self.max_seq: return '' return chromosomes[gene.seqid][start:end] else: start = max(item.transcription_stop for item in peaks) end = gene.utr_pos if end - start > self.max_seq: return '' return bio.reverse_complement( chromosomes[gene.seqid][start:end]) # Normalization files if self.norm_file: norm_file = self.norm_file else: nesoni.Norm_from_counts(self.prefix + '-norm', self.counts).run() norm_file = self.prefix + '-norm.csv' norms = io.read_grouped_table(norm_file, [('All', str)])['All'] pair_norm_names = [] pair_norms = [] for i in xrange(len(norms)): pair_norm_names.append(norms.keys()[i] + '-peak1') pair_norms.append(norms.values()[i]) for i in xrange(len(norms)): pair_norm_names.append(norms.keys()[i] + '-peak2') pair_norms.append(norms.values()[i]) io.write_grouped_csv( self.prefix + '-pairs-norm.csv', [('All', io.named_list_type(pair_norm_names)(pair_norms))], comments=['#Normalization'], ) # Read data annotations = list(annotation.read_annotations(self.parents)) if self.utrs: utrs = list(annotation.read_annotations(self.utrs)) else: utrs = [] children = list(annotation.read_annotations(self.children)) count_table = io.read_grouped_table(self.counts, [('Count', int), ('Tail_count', int), ('Tail', _float_or_none), ('Proportion', _float_or_none), ('Annotation', str)]) counts = count_table['Count'] tail_counts = count_table['Tail_count'] proportions = count_table['Proportion'] tails = count_table['Tail'] samples = counts.value_type().keys() sample_tags = {} for line in count_table.comments: if line.startswith('#sampleTags='): parts = line[len('#sampleTags='):].split(',') assert parts[0] not in sample_tags sample_tags[parts[0]] = parts for item in children: item.weight = sum(counts[item.get_id()][name] * float(norms[name]['Normalizing.multiplier']) for name in samples) parents = [] id_to_parent = {} for item in annotations: if item.type != self.parent_type: continue assert item.get_id( ) not in id_to_parent, 'Duplicate id in parent file: ' + item.get_id( ) parents.append(item) id_to_parent[item.get_id()] = item item.children = [] #item.cds = [ ] # Default utr if item.strand >= 0: item.utr_pos = item.end else: item.utr_pos = item.start if 'three_prime_UTR_start' in item.attr: if item.strand >= 0: item.utr_pos = int(item.attr['three_prime_UTR_start']) - 1 else: item.utr_pos = int(item.attr['three_prime_UTR_start']) for item in utrs: assert item.attr[ 'Parent'] in id_to_parent, 'Unknown gene ' + item.attr['Parent'] id_to_parent[item.attr['Parent']].utr_pos = ( item.start if item.strand >= 0 else item.end) for item in children: item.transcription_stop = item.end if item.strand >= 0 else item.start #End of transcription, 0-based, ie between-positions based if 'Parent' in item.attr and item.attr.get( "Relation") != "Antisense": for item_parent in item.attr['Parent'].split(','): parent = id_to_parent[item_parent] parent.children.append(item) for item in parents: item.children.sort(key=_annotation_sorter) relevant = list(item.children) if self.utr_only: #if item.strand <= 0: # relative_utr_start = item.end - int(item.attr['three_prime_UTR_start']) #else: # relative_utr_start = int(item.attr['three_prime_UTR_start'])-1 - item.start # #def relative_start(peak): # return item.end-peak.end if item.strand < 0 else peak.start-item.start #relevant = [ peak for peak in relevant if relative_start(peak) >= relative_utr_start ] #relevant = [ # peak for peak in relevant # if (peak.end >= item.utr_pos if item.strand >= 0 else peak.start <= item.utr_pos) # ] relevant = [ peak for peak in relevant if peak.attr.get("Relation") == "3'UTR" ] if self.top: relevant.sort(key=lambda peak: peak.weight, reverse=True) relevant = relevant[:self.top] relevant.sort(key=_annotation_sorter) item.relevant_children = relevant # JSON output j_data = {} j_genes = j_data['genes'] = {} j_genes['__comment__'] = 'start is 0-based' j_genes['name'] = [] j_genes['chromosome'] = [] j_genes['strand'] = [] j_genes['start'] = [] j_genes['utr'] = [] j_genes['end'] = [] j_genes['gene'] = [] j_genes['product'] = [] j_genes['peaks'] = [] j_genes['relevant_peaks'] = [] #j_genes['cds'] = [ ] #j_genes['cds_start'] = [ ] #j_genes['cds_end'] = [ ] for item in parents: j_genes['name'].append(item.get_id()) j_genes['chromosome'].append(item.seqid) j_genes['strand'].append(item.strand) j_genes['start'].append(item.start) j_genes['utr'].append(item.utr_pos) j_genes['end'].append(item.end) j_genes['gene'].append( item.attr.get('Name', item.attr.get('gene', ''))) j_genes['product'].append( item.attr.get('Product', item.attr.get('product', ''))) j_genes['peaks'].append( [item2.get_id() for item2 in item.children]) j_genes['relevant_peaks'].append( [item2.get_id() for item2 in item.relevant_children]) #j_genes['cds'].append( item.cds ) #j_genes['cds_start'].append( item.cds_start ) #j_genes['cds_end'].append( item.cds_end ) j_peaks = j_data['peaks'] = {} j_peaks['__comment__'] = 'start is 0-based' j_peaks['name'] = [] j_peaks['chromosome'] = [] j_peaks['strand'] = [] j_peaks['start'] = [] j_peaks['end'] = [] j_peaks['parents'] = [] j_peaks['counts'] = [] j_peaks['tail_lengths'] = [] j_peaks['proportion_tailed'] = [] for item in children: j_peaks['name'].append(item.get_id()) j_peaks['chromosome'].append(item.seqid) j_peaks['strand'].append(item.strand) j_peaks['start'].append(item.start) j_peaks['end'].append(item.end) j_peaks['parents'].append(item.attr['Parent'].split(',') if 'Parent' in item.attr else []) j_peaks['counts'].append(counts[item.get_id()].values()) j_peaks['tail_lengths'].append( count_table['Tail'][item.get_id()].values()) j_peaks['proportion_tailed'].append( count_table['Proportion'][item.get_id()].values()) j_samples = j_data['samples'] = {} j_samples['name'] = [] j_samples['tags'] = [] j_samples['normalizing_multiplier'] = [] for name in samples: j_samples['name'].append(name) j_samples['tags'].append(sample_tags[name]) j_samples['normalizing_multiplier'].append( float(norms[name]['Normalizing.multiplier'])) j_chromosomes = j_data['chromosomes'] = {} j_chromosomes['name'] = [] j_chromosomes['length'] = [] for name, seq in chromosomes.iteritems(): j_chromosomes['name'].append(name) j_chromosomes['length'].append(len(seq)) with open(self.prefix + '.json', 'wb') as f: json.dump(j_data, f) # Output paired peak file output_comments = ['#Counts'] output_samples = [] for item in samples: output_samples.append(item + '-peak1') output_comments.append('#sampleTags=' + ','.join([item + '-peak1', 'peak1'] + sample_tags.get(item, []))) for item in samples: output_samples.append(item + '-peak2') output_comments.append('#sampleTags=' + ','.join([item + '-peak2', 'peak2'] + sample_tags.get(item, []))) output_names = [] output_counts = [] output_tail_counts = [] output_proportions = [] output_tails = [] output_annotation_fields = [ 'gene', 'product', 'biotype', 'mean_tail_1', 'mean_tail_2', 'chromosome', 'strand', 'transcription_stops' ] #, 'interpeak_seq', ] output_annotations = [] for item in parents: peaks = item.relevant_children for i in xrange(len(peaks) - 1): for j in xrange(i + 1, len(peaks)): id_i = peaks[i].get_id() id_j = peaks[j].get_id() id_pair = item.get_id() + '-' + id_i + '-' + id_j output_names.append(id_pair) row = [] row.extend(counts[id_i].values()) row.extend(counts[id_j].values()) output_counts.append(filter(_text, row)) row = [] row.extend(tail_counts[id_i].values()) row.extend(tail_counts[id_j].values()) output_tail_counts.append(filter(_text, row)) row = [] row.extend(proportions[id_i].values()) row.extend(proportions[id_j].values()) output_proportions.append(filter(_text, row)) row = [] row.extend(tails[id_i].values()) row.extend(tails[id_j].values()) output_tails.append(filter(_text, row)) output_annotations.append([ item.attr.get('Name', item.attr.get('gene', '')), item.attr.get('Product', item.attr.get('product', '')), item.attr.get('Biotype', ''), count_table['Annotation'][id_i]['mean-tail'], count_table['Annotation'][id_j]['mean-tail'], item.seqid, str(item.strand), '%d, %d' % (peaks[i].transcription_stop, peaks[j].transcription_stop), #get_interpeak_seq([peaks[i],peaks[j]]), ]) #output_count_table = io.named_matrix_type(output_names,output_samples)(output_counts) io.write_grouped_csv( self.prefix + '-pairs.csv', [ ('Count', io.named_matrix_type(output_names, output_samples)(output_counts)), ('Tail_count', io.named_matrix_type(output_names, output_samples)(output_tail_counts)), ('Proportion', io.named_matrix_type(output_names, output_samples)(output_proportions)), ('Tail', io.named_matrix_type(output_names, output_samples)(output_tails)), ('Annotation', io.named_matrix_type( output_names, output_annotation_fields)(output_annotations)), ], comments=output_comments, )
def run(self): data = io.read_grouped_table( self.counts, [('Count', str), ('Annotation', str), ('Tail_count', str), ('Tail', str), ('Proportion', str)], 'Count', ) features = data['Count'].keys() samples = data['Count'].value_type().keys() tags = {} for sample in samples: tags[sample] = [sample] for line in data.comments: if line.startswith('#sampleTags='): parts = line[len('#sampleTags='):].split(',') tags[parts[0]] = parts group_names = [] groups = [] group_tags = [] for item in self.groups: select = selection.term_specification(item) name = selection.term_name(item) group = [ item for item in samples if selection.matches(select, tags[item]) ] assert group, 'Empty group: ' + name this_group_tags = [name] for tag in tags[group[0]]: if tag == name: continue for item in group[1:]: for item2 in tags[item]: if tag not in item2: break else: this_group_tags.append(tag) group_names.append(name) groups.append(group) group_tags.append(this_group_tags) result = io.Grouped_table() result.comments = ['#Counts'] for item in group_tags: result.comments.append('#sampleTags=' + ','.join(item)) count = [] tail_count = [] tail = [] proportion = [] for feature in features: this_count = [] this_tail_count = [] this_tail = [] this_proportion = [] for group in groups: this_this_count = [] this_this_tail_count = [] this_this_tail = [] this_this_proportion = [] for sample in group: this_this_count.append(int(data['Count'][feature][sample])) this_this_tail_count.append( int(data['Tail_count'][feature][sample])) item = data['Tail'][feature][sample] if item != 'NA': this_this_tail.append(float(item)) item = data['Proportion'][feature][sample] if item != 'NA': this_this_proportion.append(float(item)) this_count.append(str(sum(this_this_count))) this_tail_count.append(str(sum(this_this_tail_count))) this_tail.append( str(sum(this_this_tail) / len(this_this_tail)) if this_this_tail else 'NA') this_proportion.append( str(sum(this_this_proportion) / len(this_this_proportion) ) if this_this_proportion else 'NA') count.append(this_count) tail_count.append(this_tail_count) tail.append(this_tail) proportion.append(this_proportion) matrix = io.named_matrix_type(features, group_names) result['Count'] = matrix(count) result['Annotation'] = data['Annotation'] result['Tail_count'] = matrix(tail_count) result['Tail'] = matrix(tail) result['Proportion'] = matrix(proportion) result.write_csv(self.prefix + '.csv')