def main(): # Arguments input_fname, out_fname = sys.argv[1:] # Do conversion. index = Indexes() offset = 0 reader_wrapper = GFFReaderWrapper(fileinput.FileInput(input_fname), fix_strand=True) for feature in list(reader_wrapper): # Add feature; index expects BED coordinates. if isinstance(feature, GenomicInterval): convert_gff_coords_to_bed(feature) index.add(feature.chrom, feature.start, feature.end, offset) # Always increment offset, even if feature is not an interval and hence # not included in the index. offset += feature.raw_size index.write(open(out_fname, "w"))
def main(): # Arguments input_fname, out_fname = sys.argv[1:] # Do conversion. index = Indexes() offset = 0 reader_wrapper = GFFReaderWrapper(fileinput.FileInput(input_fname), fix_strand=True) for feature in list(reader_wrapper): # Add feature; index expects BED coordinates. if isinstance(feature, GenomicInterval): convert_gff_coords_to_bed(feature) index.add(feature.chrom, feature.start, feature.end, offset) # Always increment offset, even if feature is not an interval and hence # not included in the index. offset += feature.raw_size index.write(open(out_fname, "wb"))
def main(): # Process arguments. parser = optparse.OptionParser() parser.add_option( '-F', '--format', dest="input_format" ) (options, args) = parser.parse_args() in_fname, out_fname = args input_format = options.input_format.lower() # Create dict of name-location pairings. name_loc_dict = {} if input_format in [ 'gff', 'gtf' ]: # GTF/GFF format # Create reader. if input_format == 'gff': in_reader = GFFReaderWrapper( open( in_fname, 'r' ) ) else: # input_format == 'gtf' in_reader = read_unordered_gtf( open( in_fname, 'r' ) ) for feature in in_reader: if isinstance( feature, Comment ): continue for name in feature.attributes: val = feature.attributes[ name ] try: float( val ) continue except: convert_gff_coords_to_bed( feature ) # Value is not a number, so it can be indexed. if val not in name_loc_dict: # Value is not in dictionary. name_loc_dict[ val ] = { 'contig': feature.chrom, 'start': feature.start, 'end': feature.end } else: # Value already in dictionary, so update dictionary. loc = name_loc_dict[ val ] if feature.start < loc[ 'start' ]: loc[ 'start' ] = feature.start if feature.end > loc[ 'end' ]: loc[ 'end' ] = feature.end elif input_format == 'bed': # BED format. for line in open( in_fname, 'r' ): # Ignore track lines. if line.startswith("track"): continue fields = line.split() # Ignore lines with no feature name. if len( fields ) < 4: continue # Process line name_loc_dict[ fields[3] ] = { 'contig': fields[0], 'start': int( fields[1] ), 'end': int( fields[2] ) } # Create sorted list of entries. out = open( out_fname, 'w' ) max_len = 0 entries = [] for name in sorted( name_loc_dict.iterkeys() ): loc = name_loc_dict[ name ] entry = '%s\t%s\t%s' % ( name.lower(), name, '%s:%i-%i' % ( loc[ 'contig' ], loc[ 'start' ], loc[ 'end' ] ) ) if len( entry ) > max_len: max_len = len( entry ) entries.append( entry ) # Write padded entries. out.write( str( max_len + 1 ).ljust( max_len ) + '\n' ) for entry in entries: out.write( entry.ljust( max_len ) + '\n' ) out.close()
twobitfile = None file_iterator = open(input_filename) if gff_format and interpret_features: file_iterator = gff_util.GFFReaderWrapper(file_iterator, fix_strand=False) line_count = 1 for feature in file_iterator: # Ignore comments, headers. if isinstance(feature, (Header, Comment)): line_count += 1 continue name = "" if gff_format and interpret_features: # Processing features. gff_util.convert_gff_coords_to_bed(feature) chrom = feature.chrom start = feature.start end = feature.end strand = feature.strand else: # Processing lines, either interval or GFF format. line = feature.rstrip('\r\n') if line and not line.startswith("#"): fields = line.split('\t') try: chrom = fields[chrom_col] start = int(fields[start_col]) end = int(fields[end_col]) if name_col: name = fields[name_col]
def __main__(): # # Parse options, args. # options, args = doc_optparse.parse(__doc__) try: if len(options.cols.split(',')) == 5: # BED file chrom_col, start_col, end_col, strand_col, name_col = parse_cols_arg( options.cols) else: # gff file chrom_col, start_col, end_col, strand_col = parse_cols_arg( options.cols) name_col = False dbkey = options.dbkey output_format = options.output_format gff_format = options.gff interpret_features = options.interpret_features GALAXY_DATA_INDEX_DIR = options.GALAXY_DATA_INDEX_DIR fasta_file = options.fasta input_filename, output_filename = args except: doc_optparse.exception() includes_strand_col = strand_col >= 0 strand = None nibs = {} # # Set path to sequence data. # if fasta_file: # Need to create 2bit file from fasta file. try: seq_path = tempfile.NamedTemporaryFile(dir=".").name cmd = "faToTwoBit %s %s" % (fasta_file, seq_path) tmp_name = tempfile.NamedTemporaryFile(dir=".").name tmp_stderr = open(tmp_name, 'wb') proc = subprocess.Popen(args=cmd, shell=True, stderr=tmp_stderr.fileno()) returncode = proc.wait() tmp_stderr.close() # Get stderr, allowing for case where it's very large. tmp_stderr = open(tmp_name, 'rb') stderr = '' buffsize = 1048576 try: while True: stderr += tmp_stderr.read(buffsize) if not stderr or len(stderr) % buffsize != 0: break except OverflowError: pass tmp_stderr.close() # Error checking. if returncode != 0: raise Exception(stderr) except Exception as e: stop_err('Error running faToTwoBit. ' + str(e)) else: seq_path = check_seq_file(dbkey, GALAXY_DATA_INDEX_DIR) if not os.path.exists(seq_path): # If this occurs, we need to fix the metadata validator. stop_err( "No sequences are available for '%s', request them by reporting this error." % dbkey) # # Fetch sequences. # # Get feature's line(s). def get_lines(feature): if isinstance(feature, gff_util.GFFFeature): return feature.lines() else: return [feature.rstrip('\r\n')] skipped_lines = 0 first_invalid_line = 0 invalid_lines = [] fout = open(output_filename, "w") warnings = [] warning = '' twobitfile = None file_iterator = open(input_filename) if gff_format and interpret_features: file_iterator = gff_util.GFFReaderWrapper(file_iterator, fix_strand=False) line_count = 1 for feature in file_iterator: # Ignore comments, headers. if isinstance(feature, (Header, Comment)): line_count += 1 continue name = "" if gff_format and interpret_features: # Processing features. gff_util.convert_gff_coords_to_bed(feature) chrom = feature.chrom start = feature.start end = feature.end strand = feature.strand else: # Processing lines, either interval or GFF format. line = feature.rstrip('\r\n') if line and not line.startswith("#"): fields = line.split('\t') try: chrom = fields[chrom_col] start = int(fields[start_col]) end = int(fields[end_col]) if name_col: name = fields[name_col] if gff_format: start, end = gff_util.convert_gff_coords_to_bed( [start, end]) if includes_strand_col: strand = fields[strand_col] except: warning = "Invalid chrom, start or end column values. " warnings.append(warning) if not invalid_lines: invalid_lines = get_lines(feature) first_invalid_line = line_count skipped_lines += len(invalid_lines) continue if start > end: warning = "Invalid interval, start '%d' > end '%d'. " % ( start, end) warnings.append(warning) if not invalid_lines: invalid_lines = get_lines(feature) first_invalid_line = line_count skipped_lines += len(invalid_lines) continue if strand not in ['+', '-']: strand = '+' sequence = '' else: continue # Open sequence file and get sequence for feature/interval. if seq_path and os.path.exists("%s/%s.nib" % (seq_path, chrom)): # TODO: improve support for GFF-nib interaction. if chrom in nibs: nib = nibs[chrom] else: nibs[chrom] = nib = bx.seq.nib.NibFile( open("%s/%s.nib" % (seq_path, chrom))) try: sequence = nib.get(start, end - start) except Exception as e: warning = "Unable to fetch the sequence from '%d' to '%d' for build '%s'. " % ( start, end - start, dbkey) warnings.append(warning) if not invalid_lines: invalid_lines = get_lines(feature) first_invalid_line = line_count skipped_lines += len(invalid_lines) continue elif seq_path and os.path.isfile(seq_path): if not (twobitfile): twobitfile = bx.seq.twobit.TwoBitFile(open(seq_path)) try: if options.gff and interpret_features: # Create sequence from intervals within a feature. sequence = '' for interval in feature.intervals: sequence += twobitfile[ interval.chrom][interval.start:interval.end] else: sequence = twobitfile[chrom][start:end] except: warning = "Unable to fetch the sequence from '%d' to '%d' for chrom '%s'. " % ( start, end - start, chrom) warnings.append(warning) if not invalid_lines: invalid_lines = get_lines(feature) first_invalid_line = line_count skipped_lines += len(invalid_lines) continue else: warning = "Chromosome by name '%s' was not found for build '%s'. " % ( chrom, dbkey) warnings.append(warning) if not invalid_lines: invalid_lines = get_lines(feature) first_invalid_line = line_count skipped_lines += len(invalid_lines) continue if sequence == '': warning = "Chrom: '%s', start: '%s', end: '%s' is either invalid or not present in build '%s'. " % \ ( chrom, start, end, dbkey ) warnings.append(warning) if not invalid_lines: invalid_lines = get_lines(feature) first_invalid_line = line_count skipped_lines += len(invalid_lines) continue if includes_strand_col and strand == "-": sequence = reverse_complement(sequence) if output_format == "fasta": l = len(sequence) c = 0 if gff_format: start, end = gff_util.convert_bed_coords_to_gff([start, end]) fields = [dbkey, str(chrom), str(start), str(end), strand] meta_data = "_".join(fields) if name.strip(): fout.write(">%s %s\n" % (meta_data, name)) else: fout.write(">%s\n" % meta_data) while c < l: b = min(c + 50, l) fout.write("%s\n" % str(sequence[c:b])) c = b else: # output_format == "interval" if gff_format and interpret_features: # TODO: need better GFF Reader to capture all information needed # to produce this line. meta_data = "\t".join([ feature.chrom, "galaxy_extract_genomic_dna", "interval", str(feature.start), str(feature.end), feature.score, feature.strand, ".", gff_util.gff_attributes_to_str(feature.attributes, "GTF") ]) else: meta_data = "\t".join(fields) if gff_format: format_str = "%s seq \"%s\";\n" else: format_str = "%s\t%s\n" fout.write(format_str % (meta_data, str(sequence))) # Update line count. if isinstance(feature, gff_util.GFFFeature): line_count += len(feature.intervals) else: line_count += 1 fout.close() if warnings: warn_msg = "%d warnings, 1st is: " % len(warnings) warn_msg += warnings[0] print warn_msg if skipped_lines: # Error message includes up to the first 10 skipped lines. print 'Skipped %d invalid lines, 1st is #%d, "%s"' % ( skipped_lines, first_invalid_line, '\n'.join(invalid_lines[:10])) # Clean up temp file. if fasta_file: os.remove(seq_path) os.remove(tmp_name)
def __main__(): # # Parse options, args. # options, args = doc_optparse.parse( __doc__ ) try: if len(options.cols.split(',')) == 5: # BED file chrom_col, start_col, end_col, strand_col, name_col = parse_cols_arg( options.cols ) else: # gff file chrom_col, start_col, end_col, strand_col = parse_cols_arg( options.cols ) name_col = False dbkey = options.dbkey output_format = options.output_format gff_format = options.gff interpret_features = options.interpret_features GALAXY_DATA_INDEX_DIR = options.GALAXY_DATA_INDEX_DIR fasta_file = options.fasta input_filename, output_filename = args except: doc_optparse.exception() includes_strand_col = strand_col >= 0 strand = None nibs = {} # # Set path to sequence data. # if fasta_file: # Need to create 2bit file from fasta file. try: seq_path = tempfile.NamedTemporaryFile( dir="." ).name cmd = "faToTwoBit %s %s" % ( fasta_file, seq_path ) tmp_name = tempfile.NamedTemporaryFile( dir="." ).name tmp_stderr = open( tmp_name, 'wb' ) proc = subprocess.Popen( args=cmd, shell=True, stderr=tmp_stderr.fileno() ) returncode = proc.wait() tmp_stderr.close() # Get stderr, allowing for case where it's very large. tmp_stderr = open( tmp_name, 'rb' ) stderr = '' buffsize = 1048576 try: while True: stderr += tmp_stderr.read( buffsize ) if not stderr or len( stderr ) % buffsize != 0: break except OverflowError: pass tmp_stderr.close() # Error checking. if returncode != 0: raise Exception(stderr) except Exception as e: stop_err( 'Error running faToTwoBit. ' + str( e ) ) else: seq_path = check_seq_file( dbkey, GALAXY_DATA_INDEX_DIR ) if not os.path.exists( seq_path ): # If this occurs, we need to fix the metadata validator. stop_err( "No sequences are available for '%s', request them by reporting this error." % dbkey ) # # Fetch sequences. # # Get feature's line(s). def get_lines( feature ): if isinstance( feature, gff_util.GFFFeature ): return feature.lines() else: return [ feature.rstrip( '\r\n' ) ] skipped_lines = 0 first_invalid_line = 0 invalid_lines = [] fout = open( output_filename, "w" ) warnings = [] warning = '' twobitfile = None file_iterator = open( input_filename ) if gff_format and interpret_features: file_iterator = gff_util.GFFReaderWrapper( file_iterator, fix_strand=False ) line_count = 1 for feature in file_iterator: # Ignore comments, headers. if isinstance( feature, ( Header, Comment ) ): line_count += 1 continue name = "" if gff_format and interpret_features: # Processing features. gff_util.convert_gff_coords_to_bed( feature ) chrom = feature.chrom start = feature.start end = feature.end strand = feature.strand else: # Processing lines, either interval or GFF format. line = feature.rstrip( '\r\n' ) if line and not line.startswith( "#" ): fields = line.split( '\t' ) try: chrom = fields[chrom_col] start = int( fields[start_col] ) end = int( fields[end_col] ) if name_col: name = fields[name_col] if gff_format: start, end = gff_util.convert_gff_coords_to_bed( [start, end] ) if includes_strand_col: strand = fields[strand_col] except: warning = "Invalid chrom, start or end column values. " warnings.append( warning ) if not invalid_lines: invalid_lines = get_lines( feature ) first_invalid_line = line_count skipped_lines += len( invalid_lines ) continue if start > end: warning = "Invalid interval, start '%d' > end '%d'. " % ( start, end ) warnings.append( warning ) if not invalid_lines: invalid_lines = get_lines( feature ) first_invalid_line = line_count skipped_lines += len( invalid_lines ) continue if strand not in ['+', '-']: strand = '+' sequence = '' else: continue # Open sequence file and get sequence for feature/interval. if seq_path and os.path.exists( "%s/%s.nib" % ( seq_path, chrom ) ): # TODO: improve support for GFF-nib interaction. if chrom in nibs: nib = nibs[chrom] else: nibs[chrom] = nib = bx.seq.nib.NibFile( open( "%s/%s.nib" % ( seq_path, chrom ) ) ) try: sequence = nib.get( start, end - start ) except Exception as e: warning = "Unable to fetch the sequence from '%d' to '%d' for build '%s'. " % ( start, end - start, dbkey ) warnings.append( warning ) if not invalid_lines: invalid_lines = get_lines( feature ) first_invalid_line = line_count skipped_lines += len( invalid_lines ) continue elif seq_path and os.path.isfile( seq_path ): if not(twobitfile): twobitfile = bx.seq.twobit.TwoBitFile( open( seq_path ) ) try: if options.gff and interpret_features: # Create sequence from intervals within a feature. sequence = '' for interval in feature.intervals: sequence += twobitfile[interval.chrom][interval.start:interval.end] else: sequence = twobitfile[chrom][start:end] except: warning = "Unable to fetch the sequence from '%d' to '%d' for chrom '%s'. " % ( start, end - start, chrom ) warnings.append( warning ) if not invalid_lines: invalid_lines = get_lines( feature ) first_invalid_line = line_count skipped_lines += len( invalid_lines ) continue else: warning = "Chromosome by name '%s' was not found for build '%s'. " % ( chrom, dbkey ) warnings.append( warning ) if not invalid_lines: invalid_lines = get_lines( feature ) first_invalid_line = line_count skipped_lines += len( invalid_lines ) continue if sequence == '': warning = "Chrom: '%s', start: '%s', end: '%s' is either invalid or not present in build '%s'. " % \ ( chrom, start, end, dbkey ) warnings.append( warning ) if not invalid_lines: invalid_lines = get_lines( feature ) first_invalid_line = line_count skipped_lines += len( invalid_lines ) continue if includes_strand_col and strand == "-": sequence = reverse_complement( sequence ) if output_format == "fasta": l = len( sequence ) c = 0 if gff_format: start, end = gff_util.convert_bed_coords_to_gff( [ start, end ] ) fields = [dbkey, str( chrom ), str( start ), str( end ), strand] meta_data = "_".join( fields ) if name.strip(): fout.write( ">%s %s\n" % (meta_data, name) ) else: fout.write( ">%s\n" % meta_data ) while c < l: b = min( c + 50, l ) fout.write( "%s\n" % str( sequence[c:b] ) ) c = b else: # output_format == "interval" if gff_format and interpret_features: # TODO: need better GFF Reader to capture all information needed # to produce this line. meta_data = "\t".join( [feature.chrom, "galaxy_extract_genomic_dna", "interval", str( feature.start ), str( feature.end ), feature.score, feature.strand, ".", gff_util.gff_attributes_to_str( feature.attributes, "GTF" ) ] ) else: meta_data = "\t".join( fields ) if gff_format: format_str = "%s seq \"%s\";\n" else: format_str = "%s\t%s\n" fout.write( format_str % ( meta_data, str( sequence ) ) ) # Update line count. if isinstance( feature, gff_util.GFFFeature ): line_count += len( feature.intervals ) else: line_count += 1 fout.close() if warnings: warn_msg = "%d warnings, 1st is: " % len( warnings ) warn_msg += warnings[0] print(warn_msg) if skipped_lines: # Error message includes up to the first 10 skipped lines. print('Skipped %d invalid lines, 1st is #%d, "%s"' % ( skipped_lines, first_invalid_line, '\n'.join( invalid_lines[:10] ) )) # Clean up temp file. if fasta_file: os.remove( seq_path ) os.remove( tmp_name )
warning = '' twobitfile = None file_iterator = open( input_filename ) if gff_format and interpret_features: file_iterator = gff_util.GFFReaderWrapper( file_iterator, fix_strand=False ) line_count = 1 for feature in file_iterator: # Ignore comments, headers. if isinstance( feature, ( Header, Comment ) ): line_count += 1 continue name = "" if gff_format and interpret_features: # Processing features. gff_util.convert_gff_coords_to_bed( feature ) chrom = feature.chrom start = feature.start end = feature.end strand = feature.strand else: # Processing lines, either interval or GFF format. line = feature.rstrip( '\r\n' ) if line and not line.startswith( "#" ): fields = line.split( '\t' ) try: chrom = fields[chrom_col] start = int( fields[start_col] ) end = int( fields[end_col] ) if name_col: name = fields[name_col]
def main(): # Process arguments. parser = optparse.OptionParser() parser.add_option('-F', '--format', dest="input_format") (options, args) = parser.parse_args() in_fname, out_fname = args input_format = options.input_format.lower() # Create dict of name-location pairings. name_loc_dict = {} if input_format in ['gff', 'gtf']: # GTF/GFF format # Create reader. if input_format == 'gff': in_reader = GFFReaderWrapper(open(in_fname, 'r')) else: #input_format == 'gtf' in_reader = read_unordered_gtf(open(in_fname, 'r')) for feature in in_reader: if isinstance(feature, Comment): continue for name in feature.attributes: val = feature.attributes[name] try: float(val) continue except: convert_gff_coords_to_bed(feature) # Value is not a number, so it can be indexed. if val not in name_loc_dict: # Value is not in dictionary. name_loc_dict[val] = { 'contig': feature.chrom, 'start': feature.start, 'end': feature.end } else: # Value already in dictionary, so update dictionary. loc = name_loc_dict[val] if feature.start < loc['start']: loc['start'] = feature.start if feature.end > loc['end']: loc['end'] = feature.end elif input_format == 'bed': # BED format. for line in open(in_fname, 'r'): # Ignore track lines. if line.startswith("track"): continue fields = line.split() # Ignore lines with no feature name. if len(fields) < 4: continue # Process line name_loc_dict[fields[3]] = { 'contig': fields[0], 'start': int(fields[1]), 'end': int(fields[2]) } # Create sorted list of entries. out = open(out_fname, 'w') max_len = 0 entries = [] for name in sorted(name_loc_dict.iterkeys()): loc = name_loc_dict[name] entry = '%s\t%s\t%s' % (name.lower(), name, '%s:%i-%i' % (loc['contig'], loc['start'], loc['end'])) if len(entry) > max_len: max_len = len(entry) entries.append(entry) # Write padded entries. out.write(str(max_len + 1).ljust(max_len) + '\n') for entry in entries: out.write(entry.ljust(max_len) + '\n') out.close()